aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorJustin Clark-Casey (justincc)2012-03-02 03:57:55 +0000
committerJustin Clark-Casey (justincc)2012-03-02 03:57:55 +0000
commitbe4199c3bc2439b0eb4ea5beef7d5702e55809c7 (patch)
treeab1931a28dbddaf1334d8004272eb3acbfe8b80e
parentStart by adding XAssetService as a copy of the existing AssetService. (diff)
downloadopensim-SC_OLD-be4199c3bc2439b0eb4ea5beef7d5702e55809c7.zip
opensim-SC_OLD-be4199c3bc2439b0eb4ea5beef7d5702e55809c7.tar.gz
opensim-SC_OLD-be4199c3bc2439b0eb4ea5beef7d5702e55809c7.tar.bz2
opensim-SC_OLD-be4199c3bc2439b0eb4ea5beef7d5702e55809c7.tar.xz
Make XAssetService a de-duplicating asset service.
This is an extremely crude implemenation which almost works by accident. Nevertheless it does work. It can be tested with the instructions at http://opensimulator.org/wiki/Feature_Proposals/Deduplicating_Asset_Service#Testing It does not interact at all with the existing asset service or any data stored there. This code is subject to change without notice and should not be used for anything other than gawking.
-rw-r--r--OpenSim/Data/MySQL/MySQLXAssetData.cs416
-rw-r--r--OpenSim/Data/MySQL/Resources/XAssetStore.migrations27
2 files changed, 443 insertions, 0 deletions
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
new file mode 100644
index 0000000..f15a9f3
--- /dev/null
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -0,0 +1,416 @@
1/*
2 * Copyright (c) Contributors, http://opensimulator.org/
3 * See CONTRIBUTORS.TXT for a full list of copyright holders.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of the OpenSimulator Project nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28using System;
29using System.Data;
30using System.Reflection;
31using System.Collections.Generic;
32using System.Text;
33using log4net;
34using MySql.Data.MySqlClient;
35using OpenMetaverse;
36using OpenSim.Framework;
37using OpenSim.Data;
38
39namespace OpenSim.Data.MySQL
40{
41 public class MySQLXAssetData : AssetDataBase
42 {
43 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
44
45 private string m_connectionString;
46 private object m_dbLock = new object();
47
48 protected virtual Assembly Assembly
49 {
50 get { return GetType().Assembly; }
51 }
52
53 #region IPlugin Members
54
55 public override string Version { get { return "1.0.0.0"; } }
56
57 /// <summary>
58 /// <para>Initialises Asset interface</para>
59 /// <para>
60 /// <list type="bullet">
61 /// <item>Loads and initialises the MySQL storage plugin.</item>
62 /// <item>Warns and uses the obsolete mysql_connection.ini if connect string is empty.</item>
63 /// <item>Check for migration</item>
64 /// </list>
65 /// </para>
66 /// </summary>
67 /// <param name="connect">connect string</param>
68 public override void Initialise(string connect)
69 {
70 m_connectionString = connect;
71
72 using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
73 {
74 dbcon.Open();
75 Migration m = new Migration(dbcon, Assembly, "XAssetStore");
76 m.Update();
77 }
78 }
79
80 public override void Initialise()
81 {
82 throw new NotImplementedException();
83 }
84
85 public override void Dispose() { }
86
87 /// <summary>
88 /// The name of this DB provider
89 /// </summary>
90 override public string Name
91 {
92 get { return "MySQL XAsset storage engine"; }
93 }
94
95 #endregion
96
97 #region IAssetDataPlugin Members
98
99 /// <summary>
100 /// Fetch Asset <paramref name="assetID"/> from database
101 /// </summary>
102 /// <param name="assetID">Asset UUID to fetch</param>
103 /// <returns>Return the asset</returns>
104 /// <remarks>On failure : throw an exception and attempt to reconnect to database</remarks>
105 override public AssetBase GetAsset(UUID assetID)
106 {
107 AssetBase asset = null;
108 lock (m_dbLock)
109 {
110 using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
111 {
112 dbcon.Open();
113
114 string hash = null;
115
116 using (MySqlCommand cmd = new MySqlCommand(
117 "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id",
118 dbcon))
119 {
120 cmd.Parameters.AddWithValue("?id", assetID.ToString());
121
122 try
123 {
124 using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
125 {
126 if (dbReader.Read())
127 {
128 asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString());
129 hash = (string)dbReader["hash"];
130 asset.Description = (string)dbReader["description"];
131
132 string local = dbReader["local"].ToString();
133 if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase))
134 asset.Local = true;
135 else
136 asset.Local = false;
137
138 asset.Temporary = Convert.ToBoolean(dbReader["temporary"]);
139 asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
140 }
141 }
142 }
143 catch (Exception e)
144 {
145 m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message);
146 }
147 }
148
149 if (asset == null)
150 return null;
151
152 m_log.DebugFormat(
153 "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash);
154
155 using (MySqlCommand cmd = new MySqlCommand(
156 "SELECT data FROM xassetsdata WHERE hash=?hash",
157 dbcon))
158 {
159 cmd.Parameters.AddWithValue("?hash", hash);
160
161 try
162 {
163 using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
164 {
165 if (dbReader.Read())
166 asset.Data = (byte[])dbReader["data"];
167 }
168 }
169 catch (Exception e)
170 {
171 m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message);
172 }
173 }
174 }
175 }
176
177 return asset;
178 }
179
180 /// <summary>
181 /// Create an asset in database, or update it if existing.
182 /// </summary>
183 /// <param name="asset">Asset UUID to create</param>
184 /// <remarks>On failure : Throw an exception and attempt to reconnect to database</remarks>
185 override public void StoreAsset(AssetBase asset)
186 {
187 lock (m_dbLock)
188 {
189 using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
190 {
191 dbcon.Open();
192
193 string assetName = asset.Name;
194 if (asset.Name.Length > 64)
195 {
196 assetName = asset.Name.Substring(0, 64);
197 m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add");
198 }
199
200 string assetDescription = asset.Description;
201 if (asset.Description.Length > 64)
202 {
203 assetDescription = asset.Description.Substring(0, 64);
204 m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add");
205 }
206
207 string hash = Util.SHA1Hash(asset.Data);
208
209 try
210 {
211 using (MySqlCommand cmd =
212 new MySqlCommand(
213 "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" +
214 "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)",
215 dbcon))
216 {
217 // create unix epoch time
218 int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
219 cmd.Parameters.AddWithValue("?id", asset.ID);
220 cmd.Parameters.AddWithValue("?hash", hash);
221 cmd.Parameters.AddWithValue("?name", assetName);
222 cmd.Parameters.AddWithValue("?description", assetDescription);
223 cmd.Parameters.AddWithValue("?asset_type", asset.Type);
224 cmd.Parameters.AddWithValue("?local", asset.Local);
225 cmd.Parameters.AddWithValue("?temporary", asset.Temporary);
226 cmd.Parameters.AddWithValue("?create_time", now);
227 cmd.Parameters.AddWithValue("?access_time", now);
228 cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID);
229 cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags);
230 cmd.Parameters.AddWithValue("?data", asset.Data);
231 cmd.ExecuteNonQuery();
232 cmd.Dispose();
233 }
234 }
235 catch (Exception e)
236 {
237 m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}",
238 asset.FullID, asset.Name, e.Message);
239 }
240
241 try
242 {
243 using (MySqlCommand cmd =
244 new MySqlCommand(
245 "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)",
246 dbcon))
247 {
248 cmd.Parameters.AddWithValue("?hash", hash);
249 cmd.Parameters.AddWithValue("?data", asset.Data);
250 cmd.ExecuteNonQuery();
251 cmd.Dispose();
252 }
253 }
254 catch (Exception e)
255 {
256 m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}",
257 asset.FullID, asset.Name, e.Message);
258 }
259 }
260 }
261 }
262
263// private void UpdateAccessTime(AssetBase asset)
264// {
265// lock (m_dbLock)
266// {
267// using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
268// {
269// dbcon.Open();
270// MySqlCommand cmd =
271// new MySqlCommand("update assets set access_time=?access_time where id=?id",
272// dbcon);
273//
274// // need to ensure we dispose
275// try
276// {
277// using (cmd)
278// {
279// // create unix epoch time
280// int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
281// cmd.Parameters.AddWithValue("?id", asset.ID);
282// cmd.Parameters.AddWithValue("?access_time", now);
283// cmd.ExecuteNonQuery();
284// cmd.Dispose();
285// }
286// }
287// catch (Exception e)
288// {
289// m_log.ErrorFormat(
290// "[ASSETS DB]: " +
291// "MySql failure updating access_time for asset {0} with name {1}" + Environment.NewLine + e.ToString()
292// + Environment.NewLine + "Attempting reconnection", asset.FullID, asset.Name);
293// }
294// }
295// }
296//
297// }
298
299 /// <summary>
300 /// Check if the asset exists in the database
301 /// </summary>
302 /// <param name="uuid">The asset UUID</param>
303 /// <returns>true if it exists, false otherwise.</returns>
304 override public bool ExistsAsset(UUID uuid)
305 {
306// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid);
307
308 bool assetExists = false;
309
310 lock (m_dbLock)
311 {
312 using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
313 {
314 dbcon.Open();
315 using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM xassetsmeta WHERE id=?id", dbcon))
316 {
317 cmd.Parameters.AddWithValue("?id", uuid.ToString());
318
319 try
320 {
321 using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
322 {
323 if (dbReader.Read())
324 {
325// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid);
326 assetExists = true;
327 }
328 }
329 }
330 catch (Exception e)
331 {
332 m_log.ErrorFormat(
333 "[XASSETS DB]: MySql failure fetching asset {0}" + Environment.NewLine + e.ToString(), uuid);
334 }
335 }
336 }
337 }
338
339 return assetExists;
340 }
341
342 /// <summary>
343 /// Returns a list of AssetMetadata objects. The list is a subset of
344 /// the entire data set offset by <paramref name="start" /> containing
345 /// <paramref name="count" /> elements.
346 /// </summary>
347 /// <param name="start">The number of results to discard from the total data set.</param>
348 /// <param name="count">The number of rows the returned list should contain.</param>
349 /// <returns>A list of AssetMetadata objects.</returns>
350 public override List<AssetMetadata> FetchAssetMetadataSet(int start, int count)
351 {
352 List<AssetMetadata> retList = new List<AssetMetadata>(count);
353
354 lock (m_dbLock)
355 {
356 using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
357 {
358 dbcon.Open();
359 MySqlCommand cmd = new MySqlCommand("SELECT name,description,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon);
360 cmd.Parameters.AddWithValue("?start", start);
361 cmd.Parameters.AddWithValue("?count", count);
362
363 try
364 {
365 using (MySqlDataReader dbReader = cmd.ExecuteReader())
366 {
367 while (dbReader.Read())
368 {
369 AssetMetadata metadata = new AssetMetadata();
370 metadata.Name = (string)dbReader["name"];
371 metadata.Description = (string)dbReader["description"];
372 metadata.Type = (sbyte)dbReader["asset_type"];
373 metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); // Not sure if this is correct.
374 metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
375 metadata.FullID = DBGuid.FromDB(dbReader["id"]);
376 metadata.CreatorID = dbReader["creator_id"].ToString();
377 metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]);
378
379 retList.Add(metadata);
380 }
381 }
382 }
383 catch (Exception e)
384 {
385 m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString());
386 }
387 }
388 }
389
390 return retList;
391 }
392
393 public override bool Delete(string id)
394 {
395 lock (m_dbLock)
396 {
397 using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
398 {
399 dbcon.Open();
400 MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon);
401 cmd.Parameters.AddWithValue("?id", id);
402 cmd.ExecuteNonQuery();
403
404 cmd.Dispose();
405
406 // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we
407 // keep a reference count (?)
408 }
409 }
410
411 return true;
412 }
413
414 #endregion
415 }
416} \ No newline at end of file
diff --git a/OpenSim/Data/MySQL/Resources/XAssetStore.migrations b/OpenSim/Data/MySQL/Resources/XAssetStore.migrations
new file mode 100644
index 0000000..b89eab2
--- /dev/null
+++ b/OpenSim/Data/MySQL/Resources/XAssetStore.migrations
@@ -0,0 +1,27 @@
1# -----------------
2:VERSION 1
3
4BEGIN;
5
6CREATE TABLE `xassetsmeta` (
7 `id` char(36) NOT NULL,
8 `hash` char(64) NOT NULL,
9 `name` varchar(64) NOT NULL,
10 `description` varchar(64) NOT NULL,
11 `asset_type` tinyint(4) NOT NULL,
12 `local` tinyint(1) NOT NULL,
13 `temporary` tinyint(1) NOT NULL,
14 `create_time` int(11) NOT NULL,
15 `access_time` int(11) NOT NULL,
16 `asset_flags` int(11) NOT NULL,
17 `creator_id` varchar(128) NOT NULL,
18 PRIMARY KEY (`id`)
19) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Version 1';
20
21CREATE TABLE `xassetsdata` (
22 `hash` char(64) NOT NULL,
23 `data` longblob NOT NULL,
24 PRIMARY KEY (`hash`)
25) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Version 1';
26
27COMMIT; \ No newline at end of file