From be4199c3bc2439b0eb4ea5beef7d5702e55809c7 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Fri, 2 Mar 2012 03:57:55 +0000
Subject: Make XAssetService a de-duplicating asset service.
This is an extremely crude implemenation which almost works by accident. Nevertheless it does work.
It can be tested with the instructions at http://opensimulator.org/wiki/Feature_Proposals/Deduplicating_Asset_Service#Testing
It does not interact at all with the existing asset service or any data stored there.
This code is subject to change without notice and should not be used for anything other than gawking.
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 416 ++++++++++++++++++++++++++++++++++
1 file changed, 416 insertions(+)
create mode 100644 OpenSim/Data/MySQL/MySQLXAssetData.cs
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
new file mode 100644
index 0000000..f15a9f3
--- /dev/null
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -0,0 +1,416 @@
+/*
+ * Copyright (c) Contributors, http://opensimulator.org/
+ * See CONTRIBUTORS.TXT for a full list of copyright holders.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the OpenSimulator Project nor the
+ * names of its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+using System;
+using System.Data;
+using System.Reflection;
+using System.Collections.Generic;
+using System.Text;
+using log4net;
+using MySql.Data.MySqlClient;
+using OpenMetaverse;
+using OpenSim.Framework;
+using OpenSim.Data;
+
+namespace OpenSim.Data.MySQL
+{
+ public class MySQLXAssetData : AssetDataBase
+ {
+ private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
+ private string m_connectionString;
+ private object m_dbLock = new object();
+
+ protected virtual Assembly Assembly
+ {
+ get { return GetType().Assembly; }
+ }
+
+ #region IPlugin Members
+
+ public override string Version { get { return "1.0.0.0"; } }
+
+ ///
+ /// Initialises Asset interface
+ ///
+ ///
+ /// - Loads and initialises the MySQL storage plugin.
+ /// - Warns and uses the obsolete mysql_connection.ini if connect string is empty.
+ /// - Check for migration
+ ///
+ ///
+ ///
+ /// connect string
+ public override void Initialise(string connect)
+ {
+ m_connectionString = connect;
+
+ using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+ {
+ dbcon.Open();
+ Migration m = new Migration(dbcon, Assembly, "XAssetStore");
+ m.Update();
+ }
+ }
+
+ public override void Initialise()
+ {
+ throw new NotImplementedException();
+ }
+
+ public override void Dispose() { }
+
+ ///
+ /// The name of this DB provider
+ ///
+ override public string Name
+ {
+ get { return "MySQL XAsset storage engine"; }
+ }
+
+ #endregion
+
+ #region IAssetDataPlugin Members
+
+ ///
+ /// Fetch Asset from database
+ ///
+ /// Asset UUID to fetch
+ /// Return the asset
+ /// On failure : throw an exception and attempt to reconnect to database
+ override public AssetBase GetAsset(UUID assetID)
+ {
+ AssetBase asset = null;
+ lock (m_dbLock)
+ {
+ using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+ {
+ dbcon.Open();
+
+ string hash = null;
+
+ using (MySqlCommand cmd = new MySqlCommand(
+ "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id",
+ dbcon))
+ {
+ cmd.Parameters.AddWithValue("?id", assetID.ToString());
+
+ try
+ {
+ using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
+ {
+ if (dbReader.Read())
+ {
+ asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString());
+ hash = (string)dbReader["hash"];
+ asset.Description = (string)dbReader["description"];
+
+ string local = dbReader["local"].ToString();
+ if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase))
+ asset.Local = true;
+ else
+ asset.Local = false;
+
+ asset.Temporary = Convert.ToBoolean(dbReader["temporary"]);
+ asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message);
+ }
+ }
+
+ if (asset == null)
+ return null;
+
+ m_log.DebugFormat(
+ "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash);
+
+ using (MySqlCommand cmd = new MySqlCommand(
+ "SELECT data FROM xassetsdata WHERE hash=?hash",
+ dbcon))
+ {
+ cmd.Parameters.AddWithValue("?hash", hash);
+
+ try
+ {
+ using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
+ {
+ if (dbReader.Read())
+ asset.Data = (byte[])dbReader["data"];
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message);
+ }
+ }
+ }
+ }
+
+ return asset;
+ }
+
+ ///
+ /// Create an asset in database, or update it if existing.
+ ///
+ /// Asset UUID to create
+ /// On failure : Throw an exception and attempt to reconnect to database
+ override public void StoreAsset(AssetBase asset)
+ {
+ lock (m_dbLock)
+ {
+ using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+ {
+ dbcon.Open();
+
+ string assetName = asset.Name;
+ if (asset.Name.Length > 64)
+ {
+ assetName = asset.Name.Substring(0, 64);
+ m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add");
+ }
+
+ string assetDescription = asset.Description;
+ if (asset.Description.Length > 64)
+ {
+ assetDescription = asset.Description.Substring(0, 64);
+ m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add");
+ }
+
+ string hash = Util.SHA1Hash(asset.Data);
+
+ try
+ {
+ using (MySqlCommand cmd =
+ new MySqlCommand(
+ "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" +
+ "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)",
+ dbcon))
+ {
+ // create unix epoch time
+ int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
+ cmd.Parameters.AddWithValue("?id", asset.ID);
+ cmd.Parameters.AddWithValue("?hash", hash);
+ cmd.Parameters.AddWithValue("?name", assetName);
+ cmd.Parameters.AddWithValue("?description", assetDescription);
+ cmd.Parameters.AddWithValue("?asset_type", asset.Type);
+ cmd.Parameters.AddWithValue("?local", asset.Local);
+ cmd.Parameters.AddWithValue("?temporary", asset.Temporary);
+ cmd.Parameters.AddWithValue("?create_time", now);
+ cmd.Parameters.AddWithValue("?access_time", now);
+ cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID);
+ cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags);
+ cmd.Parameters.AddWithValue("?data", asset.Data);
+ cmd.ExecuteNonQuery();
+ cmd.Dispose();
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}",
+ asset.FullID, asset.Name, e.Message);
+ }
+
+ try
+ {
+ using (MySqlCommand cmd =
+ new MySqlCommand(
+ "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)",
+ dbcon))
+ {
+ cmd.Parameters.AddWithValue("?hash", hash);
+ cmd.Parameters.AddWithValue("?data", asset.Data);
+ cmd.ExecuteNonQuery();
+ cmd.Dispose();
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}",
+ asset.FullID, asset.Name, e.Message);
+ }
+ }
+ }
+ }
+
+// private void UpdateAccessTime(AssetBase asset)
+// {
+// lock (m_dbLock)
+// {
+// using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+// {
+// dbcon.Open();
+// MySqlCommand cmd =
+// new MySqlCommand("update assets set access_time=?access_time where id=?id",
+// dbcon);
+//
+// // need to ensure we dispose
+// try
+// {
+// using (cmd)
+// {
+// // create unix epoch time
+// int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
+// cmd.Parameters.AddWithValue("?id", asset.ID);
+// cmd.Parameters.AddWithValue("?access_time", now);
+// cmd.ExecuteNonQuery();
+// cmd.Dispose();
+// }
+// }
+// catch (Exception e)
+// {
+// m_log.ErrorFormat(
+// "[ASSETS DB]: " +
+// "MySql failure updating access_time for asset {0} with name {1}" + Environment.NewLine + e.ToString()
+// + Environment.NewLine + "Attempting reconnection", asset.FullID, asset.Name);
+// }
+// }
+// }
+//
+// }
+
+ ///
+ /// Check if the asset exists in the database
+ ///
+ /// The asset UUID
+ /// true if it exists, false otherwise.
+ override public bool ExistsAsset(UUID uuid)
+ {
+// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid);
+
+ bool assetExists = false;
+
+ lock (m_dbLock)
+ {
+ using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+ {
+ dbcon.Open();
+ using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM xassetsmeta WHERE id=?id", dbcon))
+ {
+ cmd.Parameters.AddWithValue("?id", uuid.ToString());
+
+ try
+ {
+ using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
+ {
+ if (dbReader.Read())
+ {
+// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid);
+ assetExists = true;
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.ErrorFormat(
+ "[XASSETS DB]: MySql failure fetching asset {0}" + Environment.NewLine + e.ToString(), uuid);
+ }
+ }
+ }
+ }
+
+ return assetExists;
+ }
+
+ ///
+ /// Returns a list of AssetMetadata objects. The list is a subset of
+ /// the entire data set offset by containing
+ /// elements.
+ ///
+ /// The number of results to discard from the total data set.
+ /// The number of rows the returned list should contain.
+ /// A list of AssetMetadata objects.
+ public override List FetchAssetMetadataSet(int start, int count)
+ {
+ List retList = new List(count);
+
+ lock (m_dbLock)
+ {
+ using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+ {
+ dbcon.Open();
+ MySqlCommand cmd = new MySqlCommand("SELECT name,description,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon);
+ cmd.Parameters.AddWithValue("?start", start);
+ cmd.Parameters.AddWithValue("?count", count);
+
+ try
+ {
+ using (MySqlDataReader dbReader = cmd.ExecuteReader())
+ {
+ while (dbReader.Read())
+ {
+ AssetMetadata metadata = new AssetMetadata();
+ metadata.Name = (string)dbReader["name"];
+ metadata.Description = (string)dbReader["description"];
+ metadata.Type = (sbyte)dbReader["asset_type"];
+ metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); // Not sure if this is correct.
+ metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
+ metadata.FullID = DBGuid.FromDB(dbReader["id"]);
+ metadata.CreatorID = dbReader["creator_id"].ToString();
+ metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]);
+
+ retList.Add(metadata);
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString());
+ }
+ }
+ }
+
+ return retList;
+ }
+
+ public override bool Delete(string id)
+ {
+ lock (m_dbLock)
+ {
+ using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
+ {
+ dbcon.Open();
+ MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon);
+ cmd.Parameters.AddWithValue("?id", id);
+ cmd.ExecuteNonQuery();
+
+ cmd.Dispose();
+
+ // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we
+ // keep a reference count (?)
+ }
+ }
+
+ return true;
+ }
+
+ #endregion
+ }
+}
\ No newline at end of file
--
cgit v1.1
From e81b3502ef0fef2b5f449b52ea2f016861aa23f4 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Fri, 2 Mar 2012 23:26:03 +0000
Subject: Make xassetservice execute one query to retrieve the asset, not two
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 32 ++++----------------------------
1 file changed, 4 insertions(+), 28 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index f15a9f3..0dadf5e 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -104,6 +104,8 @@ namespace OpenSim.Data.MySQL
/// On failure : throw an exception and attempt to reconnect to database
override public AssetBase GetAsset(UUID assetID)
{
+// m_log.DebugFormat("[MYSQL XASSET DATA]: Looking for asset {0}", assetID);
+
AssetBase asset = null;
lock (m_dbLock)
{
@@ -114,7 +116,7 @@ namespace OpenSim.Data.MySQL
string hash = null;
using (MySqlCommand cmd = new MySqlCommand(
- "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id",
+ "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id",
dbcon))
{
cmd.Parameters.AddWithValue("?id", assetID.ToString());
@@ -126,7 +128,7 @@ namespace OpenSim.Data.MySQL
if (dbReader.Read())
{
asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString());
- hash = (string)dbReader["hash"];
+ asset.Data = (byte[])dbReader["data"];
asset.Description = (string)dbReader["description"];
string local = dbReader["local"].ToString();
@@ -145,32 +147,6 @@ namespace OpenSim.Data.MySQL
m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message);
}
}
-
- if (asset == null)
- return null;
-
- m_log.DebugFormat(
- "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash);
-
- using (MySqlCommand cmd = new MySqlCommand(
- "SELECT data FROM xassetsdata WHERE hash=?hash",
- dbcon))
- {
- cmd.Parameters.AddWithValue("?hash", hash);
-
- try
- {
- using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
- {
- if (dbReader.Read())
- asset.Data = (byte[])dbReader["data"];
- }
- }
- catch (Exception e)
- {
- m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message);
- }
- }
}
}
--
cgit v1.1
From 94b323d1d87dab168cebb2235f2969c7ca159230 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Fri, 2 Mar 2012 23:41:54 +0000
Subject: Perform asset storage transactionally
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 126 +++++++++++++++++++---------------
1 file changed, 69 insertions(+), 57 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index 0dadf5e..778c7f4 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -166,71 +166,83 @@ namespace OpenSim.Data.MySQL
{
dbcon.Open();
- string assetName = asset.Name;
- if (asset.Name.Length > 64)
+ using (MySqlTransaction transaction = dbcon.BeginTransaction())
{
- assetName = asset.Name.Substring(0, 64);
- m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add");
- }
+
+ string assetName = asset.Name;
+ if (asset.Name.Length > 64)
+ {
+ assetName = asset.Name.Substring(0, 64);
+ m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add");
+ }
+
+ string assetDescription = asset.Description;
+ if (asset.Description.Length > 64)
+ {
+ assetDescription = asset.Description.Substring(0, 64);
+ m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add");
+ }
+
+ string hash = Util.SHA1Hash(asset.Data);
+
+ try
+ {
+ using (MySqlCommand cmd =
+ new MySqlCommand(
+ "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" +
+ "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)",
+ dbcon))
+ {
+ // create unix epoch time
+ int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
+ cmd.Parameters.AddWithValue("?id", asset.ID);
+ cmd.Parameters.AddWithValue("?hash", hash);
+ cmd.Parameters.AddWithValue("?name", assetName);
+ cmd.Parameters.AddWithValue("?description", assetDescription);
+ cmd.Parameters.AddWithValue("?asset_type", asset.Type);
+ cmd.Parameters.AddWithValue("?local", asset.Local);
+ cmd.Parameters.AddWithValue("?temporary", asset.Temporary);
+ cmd.Parameters.AddWithValue("?create_time", now);
+ cmd.Parameters.AddWithValue("?access_time", now);
+ cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID);
+ cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags);
+ cmd.Parameters.AddWithValue("?data", asset.Data);
+ cmd.ExecuteNonQuery();
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}",
+ asset.FullID, asset.Name, e.Message);
- string assetDescription = asset.Description;
- if (asset.Description.Length > 64)
- {
- assetDescription = asset.Description.Substring(0, 64);
- m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add");
- }
+ transaction.Rollback();
- string hash = Util.SHA1Hash(asset.Data);
+ return;
+ }
- try
- {
- using (MySqlCommand cmd =
- new MySqlCommand(
- "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" +
- "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)",
- dbcon))
+ try
{
- // create unix epoch time
- int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow);
- cmd.Parameters.AddWithValue("?id", asset.ID);
- cmd.Parameters.AddWithValue("?hash", hash);
- cmd.Parameters.AddWithValue("?name", assetName);
- cmd.Parameters.AddWithValue("?description", assetDescription);
- cmd.Parameters.AddWithValue("?asset_type", asset.Type);
- cmd.Parameters.AddWithValue("?local", asset.Local);
- cmd.Parameters.AddWithValue("?temporary", asset.Temporary);
- cmd.Parameters.AddWithValue("?create_time", now);
- cmd.Parameters.AddWithValue("?access_time", now);
- cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID);
- cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags);
- cmd.Parameters.AddWithValue("?data", asset.Data);
- cmd.ExecuteNonQuery();
- cmd.Dispose();
+ using (MySqlCommand cmd =
+ new MySqlCommand(
+ "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)",
+ dbcon))
+ {
+ cmd.Parameters.AddWithValue("?hash", hash);
+ cmd.Parameters.AddWithValue("?data", asset.Data);
+ cmd.ExecuteNonQuery();
+ }
}
- }
- catch (Exception e)
- {
- m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}",
- asset.FullID, asset.Name, e.Message);
- }
-
- try
- {
- using (MySqlCommand cmd =
- new MySqlCommand(
- "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)",
- dbcon))
+ catch (Exception e)
{
- cmd.Parameters.AddWithValue("?hash", hash);
- cmd.Parameters.AddWithValue("?data", asset.Data);
- cmd.ExecuteNonQuery();
- cmd.Dispose();
+ m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}",
+ asset.FullID, asset.Name, e.Message);
+
+ transaction.Rollback();
+
+ return;
}
- }
- catch (Exception e)
- {
- m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}",
- asset.FullID, asset.Name, e.Message);
+
+ transaction.Commit();
}
}
}
--
cgit v1.1
From 2535a4cafc45863a9f6bd4d30b90248014a6c2c2 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Sat, 3 Mar 2012 00:05:02 +0000
Subject: If asset data already exists with the required hash then don't
rewrite it
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 76 +++++++++++++++++++++++++++--------
1 file changed, 59 insertions(+), 17 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index 778c7f4..748578b 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -168,7 +168,6 @@ namespace OpenSim.Data.MySQL
using (MySqlTransaction transaction = dbcon.BeginTransaction())
{
-
string assetName = asset.Name;
if (asset.Name.Length > 64)
{
@@ -220,26 +219,29 @@ namespace OpenSim.Data.MySQL
return;
}
- try
+ if (!ExistsData(dbcon, transaction, hash))
{
- using (MySqlCommand cmd =
- new MySqlCommand(
- "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)",
- dbcon))
+ try
{
- cmd.Parameters.AddWithValue("?hash", hash);
- cmd.Parameters.AddWithValue("?data", asset.Data);
- cmd.ExecuteNonQuery();
+ using (MySqlCommand cmd =
+ new MySqlCommand(
+ "INSERT INTO xassetsdata(hash, data) VALUES(?hash, ?data)",
+ dbcon))
+ {
+ cmd.Parameters.AddWithValue("?hash", hash);
+ cmd.Parameters.AddWithValue("?data", asset.Data);
+ cmd.ExecuteNonQuery();
+ }
}
- }
- catch (Exception e)
- {
- m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}",
- asset.FullID, asset.Name, e.Message);
-
- transaction.Rollback();
+ catch (Exception e)
+ {
+ m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}",
+ asset.FullID, asset.Name, e.Message);
- return;
+ transaction.Rollback();
+
+ return;
+ }
}
transaction.Commit();
@@ -285,6 +287,46 @@ namespace OpenSim.Data.MySQL
// }
///
+ /// We assume we already have the m_dbLock.
+ ///
+ /// TODO: need to actually use the transaction.
+ ///
+ ///
+ ///
+ ///
+ private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, string hash)
+ {
+// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid);
+
+ bool exists = false;
+
+ using (MySqlCommand cmd = new MySqlCommand("SELECT hash FROM xassetsdata WHERE hash=?hash", dbcon))
+ {
+ cmd.Parameters.AddWithValue("?hash", hash);
+
+ try
+ {
+ using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow))
+ {
+ if (dbReader.Read())
+ {
+// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid);
+ exists = true;
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ m_log.ErrorFormat(
+ "[XASSETS DB]: MySql failure in ExistsData fetching hash {0}. Exception {1}{2}",
+ hash, e.Message, e.StackTrace);
+ }
+ }
+
+ return exists;
+ }
+
+ ///
/// Check if the asset exists in the database
///
/// The asset UUID
--
cgit v1.1
From 75dc8b1aedbd31f669c657ecc6beb6d8cbc9e037 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Sat, 3 Mar 2012 01:28:58 +0000
Subject: Implement basic gzip compression for xassetdata
Whether this is worthwhile is debatable since here we are not transmitting data over a network
In addition, jpeg2000 (the biggest data hog) is already a compressed image format.
May not remain.
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 51 ++++++++++++++++++++++++++++-------
1 file changed, 41 insertions(+), 10 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index 748578b..bb03871 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -26,9 +26,11 @@
*/
using System;
+using System.Collections.Generic;
using System.Data;
+using System.IO;
+using System.IO.Compression;
using System.Reflection;
-using System.Collections.Generic;
using System.Text;
using log4net;
using MySql.Data.MySqlClient;
@@ -139,6 +141,18 @@ namespace OpenSim.Data.MySQL
asset.Temporary = Convert.ToBoolean(dbReader["temporary"]);
asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
+
+ using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress))
+ {
+ MemoryStream outputStream = new MemoryStream();
+ WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue);
+// int compressedLength = asset.Data.Length;
+ asset.Data = outputStream.ToArray();
+
+// m_log.DebugFormat(
+// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}",
+// asset.ID, asset.Name, asset.Data.Length, compressedLength);
+ }
}
}
}
@@ -181,9 +195,24 @@ namespace OpenSim.Data.MySQL
assetDescription = asset.Description.Substring(0, 64);
m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add");
}
-
- string hash = Util.SHA1Hash(asset.Data);
-
+
+ byte[] compressedData;
+ MemoryStream outputStream = new MemoryStream();
+
+ using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false))
+ {
+ Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue));
+ // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream.
+ compressionStream.Close();
+ compressedData = outputStream.ToArray();
+ }
+
+ string hash = Util.SHA1Hash(compressedData);
+
+// m_log.DebugFormat(
+// "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}",
+// asset.ID, asset.Name, hash, compressedData.Length);
+
try
{
using (MySqlCommand cmd =
@@ -205,7 +234,6 @@ namespace OpenSim.Data.MySQL
cmd.Parameters.AddWithValue("?access_time", now);
cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID);
cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags);
- cmd.Parameters.AddWithValue("?data", asset.Data);
cmd.ExecuteNonQuery();
}
}
@@ -229,7 +257,7 @@ namespace OpenSim.Data.MySQL
dbcon))
{
cmd.Parameters.AddWithValue("?hash", hash);
- cmd.Parameters.AddWithValue("?data", asset.Data);
+ cmd.Parameters.AddWithValue("?data", compressedData);
cmd.ExecuteNonQuery();
}
}
@@ -422,16 +450,19 @@ namespace OpenSim.Data.MySQL
public override bool Delete(string id)
{
+// m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id);
+
lock (m_dbLock)
{
using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
{
dbcon.Open();
- MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon);
- cmd.Parameters.AddWithValue("?id", id);
- cmd.ExecuteNonQuery();
- cmd.Dispose();
+ using (MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon))
+ {
+ cmd.Parameters.AddWithValue("?id", id);
+ cmd.ExecuteNonQuery();
+ }
// TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we
// keep a reference count (?)
--
cgit v1.1
From 3780df8a329c81471c486acab7f641f7742267f4 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Sat, 3 Mar 2012 01:43:36 +0000
Subject: Make asset compression optional. Currently set to false and not
configurable from outside MySQLXAssetData.
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 40 +++++++++++++++++++++--------------
1 file changed, 24 insertions(+), 16 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index bb03871..bfc1c55 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -44,6 +44,7 @@ namespace OpenSim.Data.MySQL
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+ private bool m_enableCompression = false;
private string m_connectionString;
private object m_dbLock = new object();
@@ -142,16 +143,19 @@ namespace OpenSim.Data.MySQL
asset.Temporary = Convert.ToBoolean(dbReader["temporary"]);
asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
- using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress))
+ if (m_enableCompression)
{
- MemoryStream outputStream = new MemoryStream();
- WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue);
-// int compressedLength = asset.Data.Length;
- asset.Data = outputStream.ToArray();
-
-// m_log.DebugFormat(
-// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}",
-// asset.ID, asset.Name, asset.Data.Length, compressedLength);
+ using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress))
+ {
+ MemoryStream outputStream = new MemoryStream();
+ WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue);
+ // int compressedLength = asset.Data.Length;
+ asset.Data = outputStream.ToArray();
+
+ // m_log.DebugFormat(
+ // "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}",
+ // asset.ID, asset.Name, asset.Data.Length, compressedLength);
+ }
}
}
}
@@ -199,15 +203,19 @@ namespace OpenSim.Data.MySQL
byte[] compressedData;
MemoryStream outputStream = new MemoryStream();
- using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false))
+ if (m_enableCompression)
{
- Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue));
- // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream.
- compressionStream.Close();
- compressedData = outputStream.ToArray();
+ using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false))
+ {
+ // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue));
+ // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream.
+ compressionStream.Close();
+ compressedData = outputStream.ToArray();
+ asset.Data = compressedData;
+ }
}
- string hash = Util.SHA1Hash(compressedData);
+ string hash = Util.SHA1Hash(asset.Data);
// m_log.DebugFormat(
// "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}",
@@ -257,7 +265,7 @@ namespace OpenSim.Data.MySQL
dbcon))
{
cmd.Parameters.AddWithValue("?hash", hash);
- cmd.Parameters.AddWithValue("?data", compressedData);
+ cmd.Parameters.AddWithValue("?data", asset.Data);
cmd.ExecuteNonQuery();
}
}
--
cgit v1.1
From fd2b285b1bf35d02ce8c901eaccf0b41066fb6d6 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Mon, 5 Mar 2012 23:50:41 +0000
Subject: remove unnecessary hash local variable
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 2 --
1 file changed, 2 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index bfc1c55..0aff618 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -116,8 +116,6 @@ namespace OpenSim.Data.MySQL
{
dbcon.Open();
- string hash = null;
-
using (MySqlCommand cmd = new MySqlCommand(
"SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id",
dbcon))
--
cgit v1.1
From 441449e240ffceef4322661ad936928d98e3f724 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Tue, 6 Mar 2012 00:14:21 +0000
Subject: Switch to sha256 from sha1 in order to avoid future asset hash
collisions.
Some successful collision attacks have been carried out on sha1 with speculation that more are possible.
http://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms
No successful attacks have been shown on sha256, which makes it less likely that anybody will be able to engineer an asset hash collision in the future.
Tradeoff is more storage required for hashes, and more cpu to hash, though this is neglible compared to db operations and network access.
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 22 +++++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index 0aff618..4cb89fa 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -31,6 +31,7 @@ using System.Data;
using System.IO;
using System.IO.Compression;
using System.Reflection;
+using System.Security.Cryptography;
using System.Text;
using log4net;
using MySql.Data.MySqlClient;
@@ -44,15 +45,20 @@ namespace OpenSim.Data.MySQL
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
- private bool m_enableCompression = false;
- private string m_connectionString;
- private object m_dbLock = new object();
-
protected virtual Assembly Assembly
{
get { return GetType().Assembly; }
}
+ private bool m_enableCompression = false;
+ private string m_connectionString;
+ private object m_dbLock = new object();
+
+ ///
+ /// We can reuse this for all hashing since all methods are single-threaded through m_dbBLock
+ ///
+ private HashAlgorithm hasher = new SHA256CryptoServiceProvider();
+
#region IPlugin Members
public override string Version { get { return "1.0.0.0"; } }
@@ -213,7 +219,7 @@ namespace OpenSim.Data.MySQL
}
}
- string hash = Util.SHA1Hash(asset.Data);
+ byte[] hash = hasher.ComputeHash(asset.Data);
// m_log.DebugFormat(
// "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}",
@@ -328,7 +334,7 @@ namespace OpenSim.Data.MySQL
///
///
///
- private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, string hash)
+ private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, byte[] hash)
{
// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid);
@@ -438,7 +444,9 @@ namespace OpenSim.Data.MySQL
metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]);
metadata.FullID = DBGuid.FromDB(dbReader["id"]);
metadata.CreatorID = dbReader["creator_id"].ToString();
- metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]);
+
+ // We'll ignore this for now - it appears unused!
+// metadata.SHA1 = dbReader["hash"]);
retList.Add(metadata);
}
--
cgit v1.1
From 0cbdf9dad230e24db6cd8277511e8fcc0132cb7b Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Fri, 9 Mar 2012 00:05:34 +0000
Subject: Put big fat EXPERIMENTAL warning in xassetservice database plugin
This should not currently be used in any circumstances except for experimentation.
Database tables used by this plugin can still change at any time with no migration path.
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 10 ++++++++++
1 file changed, 10 insertions(+)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index 4cb89fa..501cf1a 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -76,6 +76,16 @@ namespace OpenSim.Data.MySQL
/// connect string
public override void Initialise(string connect)
{
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: THIS PLUGIN IS STRICTLY EXPERIMENTAL.");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: DO NOT USE FOR ANY DATA THAT YOU DO NOT MIND LOSING.");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: DATABASE TABLES CAN CHANGE AT ANY TIME, CAUSING EXISTING DATA TO BE LOST.");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************");
+ m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************");
+
m_connectionString = connect;
using (MySqlConnection dbcon = new MySqlConnection(m_connectionString))
--
cgit v1.1
From 3c5bd7c35ab24250f4f65e4ba90b3febaf5edd06 Mon Sep 17 00:00:00 2001
From: Justin Clark-Casey (justincc)
Date: Fri, 9 Mar 2012 00:16:49 +0000
Subject: minor: move some compression related var setup inside compression
if/then switch
---
OpenSim/Data/MySQL/MySQLXAssetData.cs | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
(limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs')
diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs
index 501cf1a..95ef72a 100644
--- a/OpenSim/Data/MySQL/MySQLXAssetData.cs
+++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs
@@ -214,17 +214,16 @@ namespace OpenSim.Data.MySQL
m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add");
}
- byte[] compressedData;
- MemoryStream outputStream = new MemoryStream();
-
if (m_enableCompression)
{
+ MemoryStream outputStream = new MemoryStream();
+
using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false))
{
// Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue));
// We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream.
compressionStream.Close();
- compressedData = outputStream.ToArray();
+ byte[] compressedData = outputStream.ToArray();
asset.Data = compressedData;
}
}
--
cgit v1.1