From dd63cd165631886806233958526f994baf60e855 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 02:23:35 +0000 Subject: Start by adding XAssetService as a copy of the existing AssetService. This is the start of exploring the creation of a bundled OpenSimulator asset service that does de-duplication and possibly file storage of assets. Along the lines of coyled's SRAS, but not intended to replace, merely to act as a more performant bundled default. Might end up nicking stuff from kcozen's patch at http://opensimulator.org/mantis/view.php?id=5429 More details at http://opensimulator.org/wiki/Feature_Proposals/Deduplicating_Asset_Service Feedback and discussion welcome as commits are made. --- .../Asset/LocalAssetServiceConnector.cs | 7 +- OpenSim/Services/AssetService/XAssetService.cs | 213 +++++++++++++++++++++ 2 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 OpenSim/Services/AssetService/XAssetService.cs (limited to 'OpenSim') diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs index 2e6ec90..c78915f 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs @@ -73,14 +73,17 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Asset return; } - string serviceDll = assetConfig.GetString("LocalServiceModule", - String.Empty); + string serviceDll = assetConfig.GetString("LocalServiceModule", String.Empty); if (serviceDll == String.Empty) { m_log.Error("[LOCAL ASSET SERVICES CONNECTOR]: No LocalServiceModule named in section AssetService"); return; } + else + { + m_log.DebugFormat("[LOCAL ASSET SERVICES CONNECTOR]: Loading asset service at {0}", serviceDll); + } Object[] args = new Object[] { source }; m_AssetService = ServerUtils.LoadPlugin(serviceDll, args); diff --git a/OpenSim/Services/AssetService/XAssetService.cs b/OpenSim/Services/AssetService/XAssetService.cs new file mode 100644 index 0000000..d161c58 --- /dev/null +++ b/OpenSim/Services/AssetService/XAssetService.cs @@ -0,0 +1,213 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using Nini.Config; +using log4net; +using OpenSim.Framework; +using OpenSim.Data; +using OpenSim.Services.Interfaces; +using OpenMetaverse; + +namespace OpenSim.Services.AssetService +{ + /// + /// This will be developed into a de-duplicating asset service. + /// XXX: Currently it's a just a copy of the existing AssetService. so please don't attempt to use it. + /// + public class XAssetService : AssetServiceBase, IAssetService + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + protected static XAssetService m_RootInstance; + + public XAssetService(IConfigSource config) : base(config) + { + if (m_RootInstance == null) + { + m_RootInstance = this; + + if (m_AssetLoader != null) + { + IConfig assetConfig = config.Configs["AssetService"]; + if (assetConfig == null) + throw new Exception("No AssetService configuration"); + + string loaderArgs = assetConfig.GetString("AssetLoaderArgs", + String.Empty); + + bool assetLoaderEnabled = assetConfig.GetBoolean("AssetLoaderEnabled", true); + + if (assetLoaderEnabled) + { + m_log.DebugFormat("[XASSET SERVICE]: Loading default asset set from {0}", loaderArgs); + + m_AssetLoader.ForEachDefaultXmlAsset( + loaderArgs, + delegate(AssetBase a) + { + AssetBase existingAsset = Get(a.ID); +// AssetMetadata existingMetadata = GetMetadata(a.ID); + + if (existingAsset == null || Util.SHA1Hash(existingAsset.Data) != Util.SHA1Hash(a.Data)) + { +// m_log.DebugFormat("[ASSET]: Storing {0} {1}", a.Name, a.ID); + Store(a); + } + }); + } + + m_log.Debug("[XASSET SERVICE]: Local asset service enabled"); + } + } + } + + public virtual AssetBase Get(string id) + { +// m_log.DebugFormat("[ASSET SERVICE]: Get asset for {0}", id); + + UUID assetID; + + if (!UUID.TryParse(id, out assetID)) + { + m_log.WarnFormat("[XASSET SERVICE]: Could not parse requested asset id {0}", id); + return null; + } + + try + { + return m_Database.GetAsset(assetID); + } + catch (Exception e) + { + m_log.ErrorFormat("[XASSET SERVICE]: Exception getting asset {0} {1}", assetID, e); + return null; + } + } + + public virtual AssetBase GetCached(string id) + { + return Get(id); + } + + public virtual AssetMetadata GetMetadata(string id) + { +// m_log.DebugFormat("[XASSET SERVICE]: Get asset metadata for {0}", id); + + UUID assetID; + + if (!UUID.TryParse(id, out assetID)) + return null; + + AssetBase asset = m_Database.GetAsset(assetID); + if (asset != null) + return asset.Metadata; + + return null; + } + + public virtual byte[] GetData(string id) + { +// m_log.DebugFormat("[XASSET SERVICE]: Get asset data for {0}", id); + + UUID assetID; + + if (!UUID.TryParse(id, out assetID)) + return null; + + AssetBase asset = m_Database.GetAsset(assetID); + return asset.Data; + } + + public virtual bool Get(string id, Object sender, AssetRetrieved handler) + { + //m_log.DebugFormat("[XASSET SERVICE]: Get asset async {0}", id); + + UUID assetID; + + if (!UUID.TryParse(id, out assetID)) + return false; + + AssetBase asset = m_Database.GetAsset(assetID); + + //m_log.DebugFormat("[XASSET SERVICE]: Got asset {0}", asset); + + handler(id, sender, asset); + + return true; + } + + public virtual string Store(AssetBase asset) + { + if (!m_Database.ExistsAsset(asset.FullID)) + { +// m_log.DebugFormat( +// "[XASSET SERVICE]: Storing asset {0} {1}, bytes {2}", asset.Name, asset.FullID, asset.Data.Length); + m_Database.StoreAsset(asset); + } +// else +// { +// m_log.DebugFormat( +// "[XASSET SERVICE]: Not storing asset {0} {1}, bytes {2} as it already exists", asset.Name, asset.FullID, asset.Data.Length); +// } + + return asset.ID; + } + + public bool UpdateContent(string id, byte[] data) + { + return false; + } + + public virtual bool Delete(string id) + { + m_log.DebugFormat("[XASSET SERVICE]: Deleting asset {0}", id); + UUID assetID; + if (!UUID.TryParse(id, out assetID)) + return false; + + AssetBase asset = m_Database.GetAsset(assetID); + if (asset == null) + return false; + + if ((int)(asset.Flags & AssetFlags.Maptile) != 0) + { + return m_Database.Delete(id); + } + else + { + m_log.DebugFormat("[XASSET SERVICE]: Request to delete asset {0}, but flags are not Maptile", id); + } + + return false; + } + } +} + -- cgit v1.1 From be4199c3bc2439b0eb4ea5beef7d5702e55809c7 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 03:57:55 +0000 Subject: Make XAssetService a de-duplicating asset service. This is an extremely crude implemenation which almost works by accident. Nevertheless it does work. It can be tested with the instructions at http://opensimulator.org/wiki/Feature_Proposals/Deduplicating_Asset_Service#Testing It does not interact at all with the existing asset service or any data stored there. This code is subject to change without notice and should not be used for anything other than gawking. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 416 +++++++++++++++++++++ .../Data/MySQL/Resources/XAssetStore.migrations | 27 ++ 2 files changed, 443 insertions(+) create mode 100644 OpenSim/Data/MySQL/MySQLXAssetData.cs create mode 100644 OpenSim/Data/MySQL/Resources/XAssetStore.migrations (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs new file mode 100644 index 0000000..f15a9f3 --- /dev/null +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -0,0 +1,416 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Data; +using System.Reflection; +using System.Collections.Generic; +using System.Text; +using log4net; +using MySql.Data.MySqlClient; +using OpenMetaverse; +using OpenSim.Framework; +using OpenSim.Data; + +namespace OpenSim.Data.MySQL +{ + public class MySQLXAssetData : AssetDataBase + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + private string m_connectionString; + private object m_dbLock = new object(); + + protected virtual Assembly Assembly + { + get { return GetType().Assembly; } + } + + #region IPlugin Members + + public override string Version { get { return "1.0.0.0"; } } + + /// + /// Initialises Asset interface + /// + /// + /// Loads and initialises the MySQL storage plugin. + /// Warns and uses the obsolete mysql_connection.ini if connect string is empty. + /// Check for migration + /// + /// + /// + /// connect string + public override void Initialise(string connect) + { + m_connectionString = connect; + + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + Migration m = new Migration(dbcon, Assembly, "XAssetStore"); + m.Update(); + } + } + + public override void Initialise() + { + throw new NotImplementedException(); + } + + public override void Dispose() { } + + /// + /// The name of this DB provider + /// + override public string Name + { + get { return "MySQL XAsset storage engine"; } + } + + #endregion + + #region IAssetDataPlugin Members + + /// + /// Fetch Asset from database + /// + /// Asset UUID to fetch + /// Return the asset + /// On failure : throw an exception and attempt to reconnect to database + override public AssetBase GetAsset(UUID assetID) + { + AssetBase asset = null; + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + + string hash = null; + + using (MySqlCommand cmd = new MySqlCommand( + "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id", + dbcon)) + { + cmd.Parameters.AddWithValue("?id", assetID.ToString()); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + { + asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString()); + hash = (string)dbReader["hash"]; + asset.Description = (string)dbReader["description"]; + + string local = dbReader["local"].ToString(); + if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) + asset.Local = true; + else + asset.Local = false; + + asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); + asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + } + } + } + catch (Exception e) + { + m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message); + } + } + + if (asset == null) + return null; + + m_log.DebugFormat( + "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash); + + using (MySqlCommand cmd = new MySqlCommand( + "SELECT data FROM xassetsdata WHERE hash=?hash", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + asset.Data = (byte[])dbReader["data"]; + } + } + catch (Exception e) + { + m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message); + } + } + } + } + + return asset; + } + + /// + /// Create an asset in database, or update it if existing. + /// + /// Asset UUID to create + /// On failure : Throw an exception and attempt to reconnect to database + override public void StoreAsset(AssetBase asset) + { + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + + string assetName = asset.Name; + if (asset.Name.Length > 64) + { + assetName = asset.Name.Substring(0, 64); + m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); + } + + string assetDescription = asset.Description; + if (asset.Description.Length > 64) + { + assetDescription = asset.Description.Substring(0, 64); + m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); + } + + string hash = Util.SHA1Hash(asset.Data); + + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + + "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", + dbcon)) + { + // create unix epoch time + int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); + cmd.Parameters.AddWithValue("?id", asset.ID); + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?name", assetName); + cmd.Parameters.AddWithValue("?description", assetDescription); + cmd.Parameters.AddWithValue("?asset_type", asset.Type); + cmd.Parameters.AddWithValue("?local", asset.Local); + cmd.Parameters.AddWithValue("?temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?create_time", now); + cmd.Parameters.AddWithValue("?access_time", now); + cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + cmd.Dispose(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + } + + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + cmd.Dispose(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + } + } + } + } + +// private void UpdateAccessTime(AssetBase asset) +// { +// lock (m_dbLock) +// { +// using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) +// { +// dbcon.Open(); +// MySqlCommand cmd = +// new MySqlCommand("update assets set access_time=?access_time where id=?id", +// dbcon); +// +// // need to ensure we dispose +// try +// { +// using (cmd) +// { +// // create unix epoch time +// int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); +// cmd.Parameters.AddWithValue("?id", asset.ID); +// cmd.Parameters.AddWithValue("?access_time", now); +// cmd.ExecuteNonQuery(); +// cmd.Dispose(); +// } +// } +// catch (Exception e) +// { +// m_log.ErrorFormat( +// "[ASSETS DB]: " + +// "MySql failure updating access_time for asset {0} with name {1}" + Environment.NewLine + e.ToString() +// + Environment.NewLine + "Attempting reconnection", asset.FullID, asset.Name); +// } +// } +// } +// +// } + + /// + /// Check if the asset exists in the database + /// + /// The asset UUID + /// true if it exists, false otherwise. + override public bool ExistsAsset(UUID uuid) + { +// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + + bool assetExists = false; + + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM xassetsmeta WHERE id=?id", dbcon)) + { + cmd.Parameters.AddWithValue("?id", uuid.ToString()); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + { +// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); + assetExists = true; + } + } + } + catch (Exception e) + { + m_log.ErrorFormat( + "[XASSETS DB]: MySql failure fetching asset {0}" + Environment.NewLine + e.ToString(), uuid); + } + } + } + } + + return assetExists; + } + + /// + /// Returns a list of AssetMetadata objects. The list is a subset of + /// the entire data set offset by containing + /// elements. + /// + /// The number of results to discard from the total data set. + /// The number of rows the returned list should contain. + /// A list of AssetMetadata objects. + public override List FetchAssetMetadataSet(int start, int count) + { + List retList = new List(count); + + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + MySqlCommand cmd = new MySqlCommand("SELECT name,description,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon); + cmd.Parameters.AddWithValue("?start", start); + cmd.Parameters.AddWithValue("?count", count); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader()) + { + while (dbReader.Read()) + { + AssetMetadata metadata = new AssetMetadata(); + metadata.Name = (string)dbReader["name"]; + metadata.Description = (string)dbReader["description"]; + metadata.Type = (sbyte)dbReader["asset_type"]; + metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); // Not sure if this is correct. + metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + metadata.FullID = DBGuid.FromDB(dbReader["id"]); + metadata.CreatorID = dbReader["creator_id"].ToString(); + metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]); + + retList.Add(metadata); + } + } + } + catch (Exception e) + { + m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString()); + } + } + } + + return retList; + } + + public override bool Delete(string id) + { + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon); + cmd.Parameters.AddWithValue("?id", id); + cmd.ExecuteNonQuery(); + + cmd.Dispose(); + + // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we + // keep a reference count (?) + } + } + + return true; + } + + #endregion + } +} \ No newline at end of file diff --git a/OpenSim/Data/MySQL/Resources/XAssetStore.migrations b/OpenSim/Data/MySQL/Resources/XAssetStore.migrations new file mode 100644 index 0000000..b89eab2 --- /dev/null +++ b/OpenSim/Data/MySQL/Resources/XAssetStore.migrations @@ -0,0 +1,27 @@ +# ----------------- +:VERSION 1 + +BEGIN; + +CREATE TABLE `xassetsmeta` ( + `id` char(36) NOT NULL, + `hash` char(64) NOT NULL, + `name` varchar(64) NOT NULL, + `description` varchar(64) NOT NULL, + `asset_type` tinyint(4) NOT NULL, + `local` tinyint(1) NOT NULL, + `temporary` tinyint(1) NOT NULL, + `create_time` int(11) NOT NULL, + `access_time` int(11) NOT NULL, + `asset_flags` int(11) NOT NULL, + `creator_id` varchar(128) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Version 1'; + +CREATE TABLE `xassetsdata` ( + `hash` char(64) NOT NULL, + `data` longblob NOT NULL, + PRIMARY KEY (`hash`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Version 1'; + +COMMIT; \ No newline at end of file -- cgit v1.1 From e81b3502ef0fef2b5f449b52ea2f016861aa23f4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 23:26:03 +0000 Subject: Make xassetservice execute one query to retrieve the asset, not two --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index f15a9f3..0dadf5e 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -104,6 +104,8 @@ namespace OpenSim.Data.MySQL /// On failure : throw an exception and attempt to reconnect to database override public AssetBase GetAsset(UUID assetID) { +// m_log.DebugFormat("[MYSQL XASSET DATA]: Looking for asset {0}", assetID); + AssetBase asset = null; lock (m_dbLock) { @@ -114,7 +116,7 @@ namespace OpenSim.Data.MySQL string hash = null; using (MySqlCommand cmd = new MySqlCommand( - "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id", + "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", dbcon)) { cmd.Parameters.AddWithValue("?id", assetID.ToString()); @@ -126,7 +128,7 @@ namespace OpenSim.Data.MySQL if (dbReader.Read()) { asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString()); - hash = (string)dbReader["hash"]; + asset.Data = (byte[])dbReader["data"]; asset.Description = (string)dbReader["description"]; string local = dbReader["local"].ToString(); @@ -145,32 +147,6 @@ namespace OpenSim.Data.MySQL m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message); } } - - if (asset == null) - return null; - - m_log.DebugFormat( - "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash); - - using (MySqlCommand cmd = new MySqlCommand( - "SELECT data FROM xassetsdata WHERE hash=?hash", - dbcon)) - { - cmd.Parameters.AddWithValue("?hash", hash); - - try - { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - asset.Data = (byte[])dbReader["data"]; - } - } - catch (Exception e) - { - m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message); - } - } } } -- cgit v1.1 From 94b323d1d87dab168cebb2235f2969c7ca159230 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 23:41:54 +0000 Subject: Perform asset storage transactionally --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 126 +++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 57 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 0dadf5e..778c7f4 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -166,71 +166,83 @@ namespace OpenSim.Data.MySQL { dbcon.Open(); - string assetName = asset.Name; - if (asset.Name.Length > 64) + using (MySqlTransaction transaction = dbcon.BeginTransaction()) { - assetName = asset.Name.Substring(0, 64); - m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); - } + + string assetName = asset.Name; + if (asset.Name.Length > 64) + { + assetName = asset.Name.Substring(0, 64); + m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); + } + + string assetDescription = asset.Description; + if (asset.Description.Length > 64) + { + assetDescription = asset.Description.Substring(0, 64); + m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); + } + + string hash = Util.SHA1Hash(asset.Data); + + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + + "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", + dbcon)) + { + // create unix epoch time + int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); + cmd.Parameters.AddWithValue("?id", asset.ID); + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?name", assetName); + cmd.Parameters.AddWithValue("?description", assetDescription); + cmd.Parameters.AddWithValue("?asset_type", asset.Type); + cmd.Parameters.AddWithValue("?local", asset.Local); + cmd.Parameters.AddWithValue("?temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?create_time", now); + cmd.Parameters.AddWithValue("?access_time", now); + cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); - string assetDescription = asset.Description; - if (asset.Description.Length > 64) - { - assetDescription = asset.Description.Substring(0, 64); - m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); - } + transaction.Rollback(); - string hash = Util.SHA1Hash(asset.Data); + return; + } - try - { - using (MySqlCommand cmd = - new MySqlCommand( - "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + - "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", - dbcon)) + try { - // create unix epoch time - int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - cmd.Parameters.AddWithValue("?id", asset.ID); - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?name", assetName); - cmd.Parameters.AddWithValue("?description", assetDescription); - cmd.Parameters.AddWithValue("?asset_type", asset.Type); - cmd.Parameters.AddWithValue("?local", asset.Local); - cmd.Parameters.AddWithValue("?temporary", asset.Temporary); - cmd.Parameters.AddWithValue("?create_time", now); - cmd.Parameters.AddWithValue("?access_time", now); - cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); - cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); - cmd.Dispose(); + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + } } - } - catch (Exception e) - { - m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); - } - - try - { - using (MySqlCommand cmd = - new MySqlCommand( - "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", - dbcon)) + catch (Exception e) { - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); - cmd.Dispose(); + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + + transaction.Rollback(); + + return; } - } - catch (Exception e) - { - m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); + + transaction.Commit(); } } } -- cgit v1.1 From 2535a4cafc45863a9f6bd4d30b90248014a6c2c2 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 3 Mar 2012 00:05:02 +0000 Subject: If asset data already exists with the required hash then don't rewrite it --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 76 +++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 17 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 778c7f4..748578b 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -168,7 +168,6 @@ namespace OpenSim.Data.MySQL using (MySqlTransaction transaction = dbcon.BeginTransaction()) { - string assetName = asset.Name; if (asset.Name.Length > 64) { @@ -220,26 +219,29 @@ namespace OpenSim.Data.MySQL return; } - try + if (!ExistsData(dbcon, transaction, hash)) { - using (MySqlCommand cmd = - new MySqlCommand( - "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", - dbcon)) + try { - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); + using (MySqlCommand cmd = + new MySqlCommand( + "INSERT INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + } } - } - catch (Exception e) - { - m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); - - transaction.Rollback(); + catch (Exception e) + { + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); - return; + transaction.Rollback(); + + return; + } } transaction.Commit(); @@ -285,6 +287,46 @@ namespace OpenSim.Data.MySQL // } /// + /// We assume we already have the m_dbLock. + /// + /// TODO: need to actually use the transaction. + /// + /// + /// + /// + private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, string hash) + { +// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + + bool exists = false; + + using (MySqlCommand cmd = new MySqlCommand("SELECT hash FROM xassetsdata WHERE hash=?hash", dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + { +// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); + exists = true; + } + } + } + catch (Exception e) + { + m_log.ErrorFormat( + "[XASSETS DB]: MySql failure in ExistsData fetching hash {0}. Exception {1}{2}", + hash, e.Message, e.StackTrace); + } + } + + return exists; + } + + /// /// Check if the asset exists in the database /// /// The asset UUID -- cgit v1.1 From 75dc8b1aedbd31f669c657ecc6beb6d8cbc9e037 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 3 Mar 2012 01:28:58 +0000 Subject: Implement basic gzip compression for xassetdata Whether this is worthwhile is debatable since here we are not transmitting data over a network In addition, jpeg2000 (the biggest data hog) is already a compressed image format. May not remain. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 51 ++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 10 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 748578b..bb03871 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -26,9 +26,11 @@ */ using System; +using System.Collections.Generic; using System.Data; +using System.IO; +using System.IO.Compression; using System.Reflection; -using System.Collections.Generic; using System.Text; using log4net; using MySql.Data.MySqlClient; @@ -139,6 +141,18 @@ namespace OpenSim.Data.MySQL asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + + using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) + { + MemoryStream outputStream = new MemoryStream(); + WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); +// int compressedLength = asset.Data.Length; + asset.Data = outputStream.ToArray(); + +// m_log.DebugFormat( +// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", +// asset.ID, asset.Name, asset.Data.Length, compressedLength); + } } } } @@ -181,9 +195,24 @@ namespace OpenSim.Data.MySQL assetDescription = asset.Description.Substring(0, 64); m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); } - - string hash = Util.SHA1Hash(asset.Data); - + + byte[] compressedData; + MemoryStream outputStream = new MemoryStream(); + + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + { + Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); + // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. + compressionStream.Close(); + compressedData = outputStream.ToArray(); + } + + string hash = Util.SHA1Hash(compressedData); + +// m_log.DebugFormat( +// "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", +// asset.ID, asset.Name, hash, compressedData.Length); + try { using (MySqlCommand cmd = @@ -205,7 +234,6 @@ namespace OpenSim.Data.MySQL cmd.Parameters.AddWithValue("?access_time", now); cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); - cmd.Parameters.AddWithValue("?data", asset.Data); cmd.ExecuteNonQuery(); } } @@ -229,7 +257,7 @@ namespace OpenSim.Data.MySQL dbcon)) { cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.Parameters.AddWithValue("?data", compressedData); cmd.ExecuteNonQuery(); } } @@ -422,16 +450,19 @@ namespace OpenSim.Data.MySQL public override bool Delete(string id) { +// m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id); + lock (m_dbLock) { using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon); - cmd.Parameters.AddWithValue("?id", id); - cmd.ExecuteNonQuery(); - cmd.Dispose(); + using (MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon)) + { + cmd.Parameters.AddWithValue("?id", id); + cmd.ExecuteNonQuery(); + } // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we // keep a reference count (?) -- cgit v1.1 From 3780df8a329c81471c486acab7f641f7742267f4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 3 Mar 2012 01:43:36 +0000 Subject: Make asset compression optional. Currently set to false and not configurable from outside MySQLXAssetData. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index bb03871..bfc1c55 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -44,6 +44,7 @@ namespace OpenSim.Data.MySQL { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private bool m_enableCompression = false; private string m_connectionString; private object m_dbLock = new object(); @@ -142,16 +143,19 @@ namespace OpenSim.Data.MySQL asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); - using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) + if (m_enableCompression) { - MemoryStream outputStream = new MemoryStream(); - WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); -// int compressedLength = asset.Data.Length; - asset.Data = outputStream.ToArray(); - -// m_log.DebugFormat( -// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", -// asset.ID, asset.Name, asset.Data.Length, compressedLength); + using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) + { + MemoryStream outputStream = new MemoryStream(); + WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); + // int compressedLength = asset.Data.Length; + asset.Data = outputStream.ToArray(); + + // m_log.DebugFormat( + // "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", + // asset.ID, asset.Name, asset.Data.Length, compressedLength); + } } } } @@ -199,15 +203,19 @@ namespace OpenSim.Data.MySQL byte[] compressedData; MemoryStream outputStream = new MemoryStream(); - using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + if (m_enableCompression) { - Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); - // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. - compressionStream.Close(); - compressedData = outputStream.ToArray(); + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + { + // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); + // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. + compressionStream.Close(); + compressedData = outputStream.ToArray(); + asset.Data = compressedData; + } } - string hash = Util.SHA1Hash(compressedData); + string hash = Util.SHA1Hash(asset.Data); // m_log.DebugFormat( // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", @@ -257,7 +265,7 @@ namespace OpenSim.Data.MySQL dbcon)) { cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", compressedData); + cmd.Parameters.AddWithValue("?data", asset.Data); cmd.ExecuteNonQuery(); } } -- cgit v1.1 From fd2b285b1bf35d02ce8c901eaccf0b41066fb6d6 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 5 Mar 2012 23:50:41 +0000 Subject: remove unnecessary hash local variable --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 2 -- 1 file changed, 2 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index bfc1c55..0aff618 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -116,8 +116,6 @@ namespace OpenSim.Data.MySQL { dbcon.Open(); - string hash = null; - using (MySqlCommand cmd = new MySqlCommand( "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", dbcon)) -- cgit v1.1 From 441449e240ffceef4322661ad936928d98e3f724 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 6 Mar 2012 00:14:21 +0000 Subject: Switch to sha256 from sha1 in order to avoid future asset hash collisions. Some successful collision attacks have been carried out on sha1 with speculation that more are possible. http://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms No successful attacks have been shown on sha256, which makes it less likely that anybody will be able to engineer an asset hash collision in the future. Tradeoff is more storage required for hashes, and more cpu to hash, though this is neglible compared to db operations and network access. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 22 +++++++++++++++------- .../Data/MySQL/Resources/XAssetStore.migrations | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 0aff618..4cb89fa 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -31,6 +31,7 @@ using System.Data; using System.IO; using System.IO.Compression; using System.Reflection; +using System.Security.Cryptography; using System.Text; using log4net; using MySql.Data.MySqlClient; @@ -44,15 +45,20 @@ namespace OpenSim.Data.MySQL { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private bool m_enableCompression = false; - private string m_connectionString; - private object m_dbLock = new object(); - protected virtual Assembly Assembly { get { return GetType().Assembly; } } + private bool m_enableCompression = false; + private string m_connectionString; + private object m_dbLock = new object(); + + /// + /// We can reuse this for all hashing since all methods are single-threaded through m_dbBLock + /// + private HashAlgorithm hasher = new SHA256CryptoServiceProvider(); + #region IPlugin Members public override string Version { get { return "1.0.0.0"; } } @@ -213,7 +219,7 @@ namespace OpenSim.Data.MySQL } } - string hash = Util.SHA1Hash(asset.Data); + byte[] hash = hasher.ComputeHash(asset.Data); // m_log.DebugFormat( // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", @@ -328,7 +334,7 @@ namespace OpenSim.Data.MySQL /// /// /// - private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, string hash) + private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, byte[] hash) { // m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); @@ -438,7 +444,9 @@ namespace OpenSim.Data.MySQL metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); metadata.FullID = DBGuid.FromDB(dbReader["id"]); metadata.CreatorID = dbReader["creator_id"].ToString(); - metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]); + + // We'll ignore this for now - it appears unused! +// metadata.SHA1 = dbReader["hash"]); retList.Add(metadata); } diff --git a/OpenSim/Data/MySQL/Resources/XAssetStore.migrations b/OpenSim/Data/MySQL/Resources/XAssetStore.migrations index b89eab2..d3cca5e 100644 --- a/OpenSim/Data/MySQL/Resources/XAssetStore.migrations +++ b/OpenSim/Data/MySQL/Resources/XAssetStore.migrations @@ -5,7 +5,7 @@ BEGIN; CREATE TABLE `xassetsmeta` ( `id` char(36) NOT NULL, - `hash` char(64) NOT NULL, + `hash` binary(32) NOT NULL, `name` varchar(64) NOT NULL, `description` varchar(64) NOT NULL, `asset_type` tinyint(4) NOT NULL, @@ -19,7 +19,7 @@ CREATE TABLE `xassetsmeta` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Version 1'; CREATE TABLE `xassetsdata` ( - `hash` char(64) NOT NULL, + `hash` binary(32) NOT NULL, `data` longblob NOT NULL, PRIMARY KEY (`hash`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Version 1'; -- cgit v1.1 From 9c8ef96301a0ff11dd5faa191b7fd49e7d1bf79d Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Thu, 8 Mar 2012 15:55:43 -0800 Subject: Fixes mantis #5923 --- OpenSim/Services/HypergridService/GatekeeperService.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Services/HypergridService/GatekeeperService.cs b/OpenSim/Services/HypergridService/GatekeeperService.cs index 0a59f86..4e38687 100644 --- a/OpenSim/Services/HypergridService/GatekeeperService.cs +++ b/OpenSim/Services/HypergridService/GatekeeperService.cs @@ -296,9 +296,10 @@ namespace OpenSim.Services.HypergridService aCircuit.firstname = account.FirstName; aCircuit.lastname = account.LastName; } - if (account == null && !aCircuit.lastname.StartsWith("@")) + if (account == null) { - aCircuit.firstname = aCircuit.firstname + "." + aCircuit.lastname; + if (!aCircuit.lastname.StartsWith("@")) + aCircuit.firstname = aCircuit.firstname + "." + aCircuit.lastname; try { Uri uri = new Uri(aCircuit.ServiceURLs["HomeURI"].ToString()); -- cgit v1.1 From 0cbdf9dad230e24db6cd8277511e8fcc0132cb7b Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 9 Mar 2012 00:05:34 +0000 Subject: Put big fat EXPERIMENTAL warning in xassetservice database plugin This should not currently be used in any circumstances except for experimentation. Database tables used by this plugin can still change at any time with no migration path. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 4cb89fa..501cf1a 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -76,6 +76,16 @@ namespace OpenSim.Data.MySQL /// connect string public override void Initialise(string connect) { + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: THIS PLUGIN IS STRICTLY EXPERIMENTAL."); + m_log.ErrorFormat("[MYSQL XASSETDATA]: DO NOT USE FOR ANY DATA THAT YOU DO NOT MIND LOSING."); + m_log.ErrorFormat("[MYSQL XASSETDATA]: DATABASE TABLES CAN CHANGE AT ANY TIME, CAUSING EXISTING DATA TO BE LOST."); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_connectionString = connect; using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) -- cgit v1.1 From 3c5bd7c35ab24250f4f65e4ba90b3febaf5edd06 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 9 Mar 2012 00:16:49 +0000 Subject: minor: move some compression related var setup inside compression if/then switch --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'OpenSim') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 501cf1a..95ef72a 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -214,17 +214,16 @@ namespace OpenSim.Data.MySQL m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); } - byte[] compressedData; - MemoryStream outputStream = new MemoryStream(); - if (m_enableCompression) { + MemoryStream outputStream = new MemoryStream(); + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) { // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. compressionStream.Close(); - compressedData = outputStream.ToArray(); + byte[] compressedData = outputStream.ToArray(); asset.Data = compressedData; } } -- cgit v1.1