From be4199c3bc2439b0eb4ea5beef7d5702e55809c7 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 03:57:55 +0000 Subject: Make XAssetService a de-duplicating asset service. This is an extremely crude implemenation which almost works by accident. Nevertheless it does work. It can be tested with the instructions at http://opensimulator.org/wiki/Feature_Proposals/Deduplicating_Asset_Service#Testing It does not interact at all with the existing asset service or any data stored there. This code is subject to change without notice and should not be used for anything other than gawking. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 416 ++++++++++++++++++++++++++++++++++ 1 file changed, 416 insertions(+) create mode 100644 OpenSim/Data/MySQL/MySQLXAssetData.cs (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs new file mode 100644 index 0000000..f15a9f3 --- /dev/null +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -0,0 +1,416 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Data; +using System.Reflection; +using System.Collections.Generic; +using System.Text; +using log4net; +using MySql.Data.MySqlClient; +using OpenMetaverse; +using OpenSim.Framework; +using OpenSim.Data; + +namespace OpenSim.Data.MySQL +{ + public class MySQLXAssetData : AssetDataBase + { + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + private string m_connectionString; + private object m_dbLock = new object(); + + protected virtual Assembly Assembly + { + get { return GetType().Assembly; } + } + + #region IPlugin Members + + public override string Version { get { return "1.0.0.0"; } } + + /// + /// Initialises Asset interface + /// + /// + /// Loads and initialises the MySQL storage plugin. + /// Warns and uses the obsolete mysql_connection.ini if connect string is empty. + /// Check for migration + /// + /// + /// + /// connect string + public override void Initialise(string connect) + { + m_connectionString = connect; + + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + Migration m = new Migration(dbcon, Assembly, "XAssetStore"); + m.Update(); + } + } + + public override void Initialise() + { + throw new NotImplementedException(); + } + + public override void Dispose() { } + + /// + /// The name of this DB provider + /// + override public string Name + { + get { return "MySQL XAsset storage engine"; } + } + + #endregion + + #region IAssetDataPlugin Members + + /// + /// Fetch Asset from database + /// + /// Asset UUID to fetch + /// Return the asset + /// On failure : throw an exception and attempt to reconnect to database + override public AssetBase GetAsset(UUID assetID) + { + AssetBase asset = null; + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + + string hash = null; + + using (MySqlCommand cmd = new MySqlCommand( + "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id", + dbcon)) + { + cmd.Parameters.AddWithValue("?id", assetID.ToString()); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + { + asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString()); + hash = (string)dbReader["hash"]; + asset.Description = (string)dbReader["description"]; + + string local = dbReader["local"].ToString(); + if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) + asset.Local = true; + else + asset.Local = false; + + asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); + asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + } + } + } + catch (Exception e) + { + m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message); + } + } + + if (asset == null) + return null; + + m_log.DebugFormat( + "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash); + + using (MySqlCommand cmd = new MySqlCommand( + "SELECT data FROM xassetsdata WHERE hash=?hash", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + asset.Data = (byte[])dbReader["data"]; + } + } + catch (Exception e) + { + m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message); + } + } + } + } + + return asset; + } + + /// + /// Create an asset in database, or update it if existing. + /// + /// Asset UUID to create + /// On failure : Throw an exception and attempt to reconnect to database + override public void StoreAsset(AssetBase asset) + { + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + + string assetName = asset.Name; + if (asset.Name.Length > 64) + { + assetName = asset.Name.Substring(0, 64); + m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); + } + + string assetDescription = asset.Description; + if (asset.Description.Length > 64) + { + assetDescription = asset.Description.Substring(0, 64); + m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); + } + + string hash = Util.SHA1Hash(asset.Data); + + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + + "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", + dbcon)) + { + // create unix epoch time + int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); + cmd.Parameters.AddWithValue("?id", asset.ID); + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?name", assetName); + cmd.Parameters.AddWithValue("?description", assetDescription); + cmd.Parameters.AddWithValue("?asset_type", asset.Type); + cmd.Parameters.AddWithValue("?local", asset.Local); + cmd.Parameters.AddWithValue("?temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?create_time", now); + cmd.Parameters.AddWithValue("?access_time", now); + cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + cmd.Dispose(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + } + + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + cmd.Dispose(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + } + } + } + } + +// private void UpdateAccessTime(AssetBase asset) +// { +// lock (m_dbLock) +// { +// using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) +// { +// dbcon.Open(); +// MySqlCommand cmd = +// new MySqlCommand("update assets set access_time=?access_time where id=?id", +// dbcon); +// +// // need to ensure we dispose +// try +// { +// using (cmd) +// { +// // create unix epoch time +// int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); +// cmd.Parameters.AddWithValue("?id", asset.ID); +// cmd.Parameters.AddWithValue("?access_time", now); +// cmd.ExecuteNonQuery(); +// cmd.Dispose(); +// } +// } +// catch (Exception e) +// { +// m_log.ErrorFormat( +// "[ASSETS DB]: " + +// "MySql failure updating access_time for asset {0} with name {1}" + Environment.NewLine + e.ToString() +// + Environment.NewLine + "Attempting reconnection", asset.FullID, asset.Name); +// } +// } +// } +// +// } + + /// + /// Check if the asset exists in the database + /// + /// The asset UUID + /// true if it exists, false otherwise. + override public bool ExistsAsset(UUID uuid) + { +// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + + bool assetExists = false; + + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM xassetsmeta WHERE id=?id", dbcon)) + { + cmd.Parameters.AddWithValue("?id", uuid.ToString()); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + { +// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); + assetExists = true; + } + } + } + catch (Exception e) + { + m_log.ErrorFormat( + "[XASSETS DB]: MySql failure fetching asset {0}" + Environment.NewLine + e.ToString(), uuid); + } + } + } + } + + return assetExists; + } + + /// + /// Returns a list of AssetMetadata objects. The list is a subset of + /// the entire data set offset by containing + /// elements. + /// + /// The number of results to discard from the total data set. + /// The number of rows the returned list should contain. + /// A list of AssetMetadata objects. + public override List FetchAssetMetadataSet(int start, int count) + { + List retList = new List(count); + + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + MySqlCommand cmd = new MySqlCommand("SELECT name,description,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon); + cmd.Parameters.AddWithValue("?start", start); + cmd.Parameters.AddWithValue("?count", count); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader()) + { + while (dbReader.Read()) + { + AssetMetadata metadata = new AssetMetadata(); + metadata.Name = (string)dbReader["name"]; + metadata.Description = (string)dbReader["description"]; + metadata.Type = (sbyte)dbReader["asset_type"]; + metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); // Not sure if this is correct. + metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + metadata.FullID = DBGuid.FromDB(dbReader["id"]); + metadata.CreatorID = dbReader["creator_id"].ToString(); + metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]); + + retList.Add(metadata); + } + } + } + catch (Exception e) + { + m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString()); + } + } + } + + return retList; + } + + public override bool Delete(string id) + { + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon); + cmd.Parameters.AddWithValue("?id", id); + cmd.ExecuteNonQuery(); + + cmd.Dispose(); + + // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we + // keep a reference count (?) + } + } + + return true; + } + + #endregion + } +} \ No newline at end of file -- cgit v1.1 From e81b3502ef0fef2b5f449b52ea2f016861aa23f4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 23:26:03 +0000 Subject: Make xassetservice execute one query to retrieve the asset, not two --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index f15a9f3..0dadf5e 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -104,6 +104,8 @@ namespace OpenSim.Data.MySQL /// On failure : throw an exception and attempt to reconnect to database override public AssetBase GetAsset(UUID assetID) { +// m_log.DebugFormat("[MYSQL XASSET DATA]: Looking for asset {0}", assetID); + AssetBase asset = null; lock (m_dbLock) { @@ -114,7 +116,7 @@ namespace OpenSim.Data.MySQL string hash = null; using (MySqlCommand cmd = new MySqlCommand( - "SELECT name, hash, description, asset_type, local, temporary, asset_flags, creator_id FROM xassetsmeta WHERE id=?id", + "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", dbcon)) { cmd.Parameters.AddWithValue("?id", assetID.ToString()); @@ -126,7 +128,7 @@ namespace OpenSim.Data.MySQL if (dbReader.Read()) { asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString()); - hash = (string)dbReader["hash"]; + asset.Data = (byte[])dbReader["data"]; asset.Description = (string)dbReader["description"]; string local = dbReader["local"].ToString(); @@ -145,32 +147,6 @@ namespace OpenSim.Data.MySQL m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message); } } - - if (asset == null) - return null; - - m_log.DebugFormat( - "[MYSQL XASSET DATA]: Looking for asset {0} {1} with hash {2}", asset.FullID, asset.Name, hash); - - using (MySqlCommand cmd = new MySqlCommand( - "SELECT data FROM xassetsdata WHERE hash=?hash", - dbcon)) - { - cmd.Parameters.AddWithValue("?hash", hash); - - try - { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - asset.Data = (byte[])dbReader["data"]; - } - } - catch (Exception e) - { - m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset metadata " + assetID + ": " + e.Message); - } - } } } -- cgit v1.1 From 94b323d1d87dab168cebb2235f2969c7ca159230 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 2 Mar 2012 23:41:54 +0000 Subject: Perform asset storage transactionally --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 126 +++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 57 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 0dadf5e..778c7f4 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -166,71 +166,83 @@ namespace OpenSim.Data.MySQL { dbcon.Open(); - string assetName = asset.Name; - if (asset.Name.Length > 64) + using (MySqlTransaction transaction = dbcon.BeginTransaction()) { - assetName = asset.Name.Substring(0, 64); - m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); - } + + string assetName = asset.Name; + if (asset.Name.Length > 64) + { + assetName = asset.Name.Substring(0, 64); + m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); + } + + string assetDescription = asset.Description; + if (asset.Description.Length > 64) + { + assetDescription = asset.Description.Substring(0, 64); + m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); + } + + string hash = Util.SHA1Hash(asset.Data); + + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + + "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", + dbcon)) + { + // create unix epoch time + int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); + cmd.Parameters.AddWithValue("?id", asset.ID); + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?name", assetName); + cmd.Parameters.AddWithValue("?description", assetDescription); + cmd.Parameters.AddWithValue("?asset_type", asset.Type); + cmd.Parameters.AddWithValue("?local", asset.Local); + cmd.Parameters.AddWithValue("?temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?create_time", now); + cmd.Parameters.AddWithValue("?access_time", now); + cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); - string assetDescription = asset.Description; - if (asset.Description.Length > 64) - { - assetDescription = asset.Description.Substring(0, 64); - m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); - } + transaction.Rollback(); - string hash = Util.SHA1Hash(asset.Data); + return; + } - try - { - using (MySqlCommand cmd = - new MySqlCommand( - "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + - "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", - dbcon)) + try { - // create unix epoch time - int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - cmd.Parameters.AddWithValue("?id", asset.ID); - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?name", assetName); - cmd.Parameters.AddWithValue("?description", assetDescription); - cmd.Parameters.AddWithValue("?asset_type", asset.Type); - cmd.Parameters.AddWithValue("?local", asset.Local); - cmd.Parameters.AddWithValue("?temporary", asset.Temporary); - cmd.Parameters.AddWithValue("?create_time", now); - cmd.Parameters.AddWithValue("?access_time", now); - cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); - cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); - cmd.Dispose(); + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + } } - } - catch (Exception e) - { - m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); - } - - try - { - using (MySqlCommand cmd = - new MySqlCommand( - "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", - dbcon)) + catch (Exception e) { - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); - cmd.Dispose(); + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + + transaction.Rollback(); + + return; } - } - catch (Exception e) - { - m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); + + transaction.Commit(); } } } -- cgit v1.1 From 2535a4cafc45863a9f6bd4d30b90248014a6c2c2 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 3 Mar 2012 00:05:02 +0000 Subject: If asset data already exists with the required hash then don't rewrite it --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 76 +++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 17 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 778c7f4..748578b 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -168,7 +168,6 @@ namespace OpenSim.Data.MySQL using (MySqlTransaction transaction = dbcon.BeginTransaction()) { - string assetName = asset.Name; if (asset.Name.Length > 64) { @@ -220,26 +219,29 @@ namespace OpenSim.Data.MySQL return; } - try + if (!ExistsData(dbcon, transaction, hash)) { - using (MySqlCommand cmd = - new MySqlCommand( - "replace INTO xassetsdata(hash, data) VALUES(?hash, ?data)", - dbcon)) + try { - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); + using (MySqlCommand cmd = + new MySqlCommand( + "INSERT INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.ExecuteNonQuery(); + } } - } - catch (Exception e) - { - m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); - - transaction.Rollback(); + catch (Exception e) + { + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); - return; + transaction.Rollback(); + + return; + } } transaction.Commit(); @@ -285,6 +287,46 @@ namespace OpenSim.Data.MySQL // } /// + /// We assume we already have the m_dbLock. + /// + /// TODO: need to actually use the transaction. + /// + /// + /// + /// + private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, string hash) + { +// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + + bool exists = false; + + using (MySqlCommand cmd = new MySqlCommand("SELECT hash FROM xassetsdata WHERE hash=?hash", dbcon)) + { + cmd.Parameters.AddWithValue("?hash", hash); + + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + { + if (dbReader.Read()) + { +// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); + exists = true; + } + } + } + catch (Exception e) + { + m_log.ErrorFormat( + "[XASSETS DB]: MySql failure in ExistsData fetching hash {0}. Exception {1}{2}", + hash, e.Message, e.StackTrace); + } + } + + return exists; + } + + /// /// Check if the asset exists in the database /// /// The asset UUID -- cgit v1.1 From 75dc8b1aedbd31f669c657ecc6beb6d8cbc9e037 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 3 Mar 2012 01:28:58 +0000 Subject: Implement basic gzip compression for xassetdata Whether this is worthwhile is debatable since here we are not transmitting data over a network In addition, jpeg2000 (the biggest data hog) is already a compressed image format. May not remain. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 51 ++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 10 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 748578b..bb03871 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -26,9 +26,11 @@ */ using System; +using System.Collections.Generic; using System.Data; +using System.IO; +using System.IO.Compression; using System.Reflection; -using System.Collections.Generic; using System.Text; using log4net; using MySql.Data.MySqlClient; @@ -139,6 +141,18 @@ namespace OpenSim.Data.MySQL asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + + using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) + { + MemoryStream outputStream = new MemoryStream(); + WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); +// int compressedLength = asset.Data.Length; + asset.Data = outputStream.ToArray(); + +// m_log.DebugFormat( +// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", +// asset.ID, asset.Name, asset.Data.Length, compressedLength); + } } } } @@ -181,9 +195,24 @@ namespace OpenSim.Data.MySQL assetDescription = asset.Description.Substring(0, 64); m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); } - - string hash = Util.SHA1Hash(asset.Data); - + + byte[] compressedData; + MemoryStream outputStream = new MemoryStream(); + + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + { + Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); + // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. + compressionStream.Close(); + compressedData = outputStream.ToArray(); + } + + string hash = Util.SHA1Hash(compressedData); + +// m_log.DebugFormat( +// "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", +// asset.ID, asset.Name, hash, compressedData.Length); + try { using (MySqlCommand cmd = @@ -205,7 +234,6 @@ namespace OpenSim.Data.MySQL cmd.Parameters.AddWithValue("?access_time", now); cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); - cmd.Parameters.AddWithValue("?data", asset.Data); cmd.ExecuteNonQuery(); } } @@ -229,7 +257,7 @@ namespace OpenSim.Data.MySQL dbcon)) { cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.Parameters.AddWithValue("?data", compressedData); cmd.ExecuteNonQuery(); } } @@ -422,16 +450,19 @@ namespace OpenSim.Data.MySQL public override bool Delete(string id) { +// m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id); + lock (m_dbLock) { using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon); - cmd.Parameters.AddWithValue("?id", id); - cmd.ExecuteNonQuery(); - cmd.Dispose(); + using (MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon)) + { + cmd.Parameters.AddWithValue("?id", id); + cmd.ExecuteNonQuery(); + } // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we // keep a reference count (?) -- cgit v1.1 From 3780df8a329c81471c486acab7f641f7742267f4 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Sat, 3 Mar 2012 01:43:36 +0000 Subject: Make asset compression optional. Currently set to false and not configurable from outside MySQLXAssetData. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index bb03871..bfc1c55 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -44,6 +44,7 @@ namespace OpenSim.Data.MySQL { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private bool m_enableCompression = false; private string m_connectionString; private object m_dbLock = new object(); @@ -142,16 +143,19 @@ namespace OpenSim.Data.MySQL asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); - using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) + if (m_enableCompression) { - MemoryStream outputStream = new MemoryStream(); - WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); -// int compressedLength = asset.Data.Length; - asset.Data = outputStream.ToArray(); - -// m_log.DebugFormat( -// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", -// asset.ID, asset.Name, asset.Data.Length, compressedLength); + using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) + { + MemoryStream outputStream = new MemoryStream(); + WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); + // int compressedLength = asset.Data.Length; + asset.Data = outputStream.ToArray(); + + // m_log.DebugFormat( + // "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", + // asset.ID, asset.Name, asset.Data.Length, compressedLength); + } } } } @@ -199,15 +203,19 @@ namespace OpenSim.Data.MySQL byte[] compressedData; MemoryStream outputStream = new MemoryStream(); - using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + if (m_enableCompression) { - Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); - // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. - compressionStream.Close(); - compressedData = outputStream.ToArray(); + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + { + // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); + // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. + compressionStream.Close(); + compressedData = outputStream.ToArray(); + asset.Data = compressedData; + } } - string hash = Util.SHA1Hash(compressedData); + string hash = Util.SHA1Hash(asset.Data); // m_log.DebugFormat( // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", @@ -257,7 +265,7 @@ namespace OpenSim.Data.MySQL dbcon)) { cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", compressedData); + cmd.Parameters.AddWithValue("?data", asset.Data); cmd.ExecuteNonQuery(); } } -- cgit v1.1 From fd2b285b1bf35d02ce8c901eaccf0b41066fb6d6 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 5 Mar 2012 23:50:41 +0000 Subject: remove unnecessary hash local variable --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 2 -- 1 file changed, 2 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index bfc1c55..0aff618 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -116,8 +116,6 @@ namespace OpenSim.Data.MySQL { dbcon.Open(); - string hash = null; - using (MySqlCommand cmd = new MySqlCommand( "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", dbcon)) -- cgit v1.1 From 441449e240ffceef4322661ad936928d98e3f724 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Tue, 6 Mar 2012 00:14:21 +0000 Subject: Switch to sha256 from sha1 in order to avoid future asset hash collisions. Some successful collision attacks have been carried out on sha1 with speculation that more are possible. http://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms No successful attacks have been shown on sha256, which makes it less likely that anybody will be able to engineer an asset hash collision in the future. Tradeoff is more storage required for hashes, and more cpu to hash, though this is neglible compared to db operations and network access. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 0aff618..4cb89fa 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -31,6 +31,7 @@ using System.Data; using System.IO; using System.IO.Compression; using System.Reflection; +using System.Security.Cryptography; using System.Text; using log4net; using MySql.Data.MySqlClient; @@ -44,15 +45,20 @@ namespace OpenSim.Data.MySQL { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private bool m_enableCompression = false; - private string m_connectionString; - private object m_dbLock = new object(); - protected virtual Assembly Assembly { get { return GetType().Assembly; } } + private bool m_enableCompression = false; + private string m_connectionString; + private object m_dbLock = new object(); + + /// + /// We can reuse this for all hashing since all methods are single-threaded through m_dbBLock + /// + private HashAlgorithm hasher = new SHA256CryptoServiceProvider(); + #region IPlugin Members public override string Version { get { return "1.0.0.0"; } } @@ -213,7 +219,7 @@ namespace OpenSim.Data.MySQL } } - string hash = Util.SHA1Hash(asset.Data); + byte[] hash = hasher.ComputeHash(asset.Data); // m_log.DebugFormat( // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", @@ -328,7 +334,7 @@ namespace OpenSim.Data.MySQL /// /// /// - private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, string hash) + private bool ExistsData(MySqlConnection dbcon, MySqlTransaction transaction, byte[] hash) { // m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); @@ -438,7 +444,9 @@ namespace OpenSim.Data.MySQL metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); metadata.FullID = DBGuid.FromDB(dbReader["id"]); metadata.CreatorID = dbReader["creator_id"].ToString(); - metadata.SHA1 = Encoding.Default.GetBytes((string)dbReader["hash"]); + + // We'll ignore this for now - it appears unused! +// metadata.SHA1 = dbReader["hash"]); retList.Add(metadata); } -- cgit v1.1 From 0cbdf9dad230e24db6cd8277511e8fcc0132cb7b Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 9 Mar 2012 00:05:34 +0000 Subject: Put big fat EXPERIMENTAL warning in xassetservice database plugin This should not currently be used in any circumstances except for experimentation. Database tables used by this plugin can still change at any time with no migration path. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 4cb89fa..501cf1a 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -76,6 +76,16 @@ namespace OpenSim.Data.MySQL /// connect string public override void Initialise(string connect) { + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: THIS PLUGIN IS STRICTLY EXPERIMENTAL."); + m_log.ErrorFormat("[MYSQL XASSETDATA]: DO NOT USE FOR ANY DATA THAT YOU DO NOT MIND LOSING."); + m_log.ErrorFormat("[MYSQL XASSETDATA]: DATABASE TABLES CAN CHANGE AT ANY TIME, CAUSING EXISTING DATA TO BE LOST."); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); + m_connectionString = connect; using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) -- cgit v1.1 From 3c5bd7c35ab24250f4f65e4ba90b3febaf5edd06 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 9 Mar 2012 00:16:49 +0000 Subject: minor: move some compression related var setup inside compression if/then switch --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 501cf1a..95ef72a 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -214,17 +214,16 @@ namespace OpenSim.Data.MySQL m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); } - byte[] compressedData; - MemoryStream outputStream = new MemoryStream(); - if (m_enableCompression) { + MemoryStream outputStream = new MemoryStream(); + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) { // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. compressionStream.Close(); - compressedData = outputStream.ToArray(); + byte[] compressedData = outputStream.ToArray(); asset.Data = compressedData; } } -- cgit v1.1 From e0dd38f6722e891cfc91c8d795604b7b78c0bc81 Mon Sep 17 00:00:00 2001 From: Robert Adams Date: Mon, 12 Mar 2012 10:07:04 -0700 Subject: Rename the stream extension method WebUtil.CopyTo() to WebUtil.CopyStream(). .NET 4.0 added the method Stream.CopyTo(stream, bufferSize). For .NET 3.5 and before, WebUtil defined an extension method for Stream with the signature Stream.CopyTo(stream, maxBytesToCopy). The meaning of the second parameter is different in the two forms and depending on which compiler and/or runtime you use, you could get one form or the other. Crashes ensue. This change renames the WebUtil stream copy method to something that cannot be confused with the new CopyTo method defined in .NET 4.0. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 95ef72a..06fe55a 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -162,7 +162,7 @@ namespace OpenSim.Data.MySQL using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) { MemoryStream outputStream = new MemoryStream(); - WebUtil.CopyTo(decompressionStream, outputStream, int.MaxValue); + WebUtil.CopyStream(decompressionStream, outputStream, int.MaxValue); // int compressedLength = asset.Data.Length; asset.Data = outputStream.ToArray(); -- cgit v1.1 From ab54ce1907e26935bfb847742d4f5aa95d34aca0 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Mon, 19 Mar 2012 00:18:04 +0000 Subject: Fix configuration problems where XAssetDatabasePlugin was picked up accidentally. The asset data plugin now implements IXAssetData rather than IAssetData so the ordinary AssetService should no longer pick it up. This replaces the changes in 92b1ade. There is no longer any need to adjust your StandaloneCommon.ini/Robust.ini/Robust.HG.ini files. This may explain very recent issues in the last few weeks where textures have been disappearing or turning white (as they were going to different places). Unfortunately, you will need to rollback to an earlier database backup or reupload the textures. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 06fe55a..e6ac22e 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -41,7 +41,7 @@ using OpenSim.Data; namespace OpenSim.Data.MySQL { - public class MySQLXAssetData : AssetDataBase + public class MySQLXAssetData : IXAssetDataPlugin { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); @@ -61,7 +61,7 @@ namespace OpenSim.Data.MySQL #region IPlugin Members - public override string Version { get { return "1.0.0.0"; } } + public string Version { get { return "1.0.0.0"; } } /// /// Initialises Asset interface @@ -74,7 +74,7 @@ namespace OpenSim.Data.MySQL /// /// /// connect string - public override void Initialise(string connect) + public void Initialise(string connect) { m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); m_log.ErrorFormat("[MYSQL XASSETDATA]: ***********************************************************"); @@ -96,17 +96,17 @@ namespace OpenSim.Data.MySQL } } - public override void Initialise() + public void Initialise() { throw new NotImplementedException(); } - public override void Dispose() { } + public void Dispose() { } /// /// The name of this DB provider /// - override public string Name + public string Name { get { return "MySQL XAsset storage engine"; } } @@ -121,7 +121,7 @@ namespace OpenSim.Data.MySQL /// Asset UUID to fetch /// Return the asset /// On failure : throw an exception and attempt to reconnect to database - override public AssetBase GetAsset(UUID assetID) + public AssetBase GetAsset(UUID assetID) { // m_log.DebugFormat("[MYSQL XASSET DATA]: Looking for asset {0}", assetID); @@ -190,7 +190,7 @@ namespace OpenSim.Data.MySQL /// /// Asset UUID to create /// On failure : Throw an exception and attempt to reconnect to database - override public void StoreAsset(AssetBase asset) + public void StoreAsset(AssetBase asset) { lock (m_dbLock) { @@ -380,7 +380,7 @@ namespace OpenSim.Data.MySQL /// /// The asset UUID /// true if it exists, false otherwise. - override public bool ExistsAsset(UUID uuid) + public bool ExistsAsset(UUID uuid) { // m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); @@ -426,7 +426,7 @@ namespace OpenSim.Data.MySQL /// The number of results to discard from the total data set. /// The number of rows the returned list should contain. /// A list of AssetMetadata objects. - public override List FetchAssetMetadataSet(int start, int count) + public List FetchAssetMetadataSet(int start, int count) { List retList = new List(count); @@ -471,7 +471,7 @@ namespace OpenSim.Data.MySQL return retList; } - public override bool Delete(string id) + public bool Delete(string id) { // m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id); -- cgit v1.1 From 74916ed777d811088344077b08a190ff17129c70 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Wed, 27 Feb 2013 21:35:54 +0000 Subject: Add more information to warnings logged when asset names and descriptions have to be truncated for database storage On balance, I still think this is useful because asset names and descriptions can sometimes be helpful in determining what things are. Even though they are never subsequently (inventory names/descriptions are always used instead). --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index e6ac22e..c2282c8 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -204,14 +204,18 @@ namespace OpenSim.Data.MySQL if (asset.Name.Length > 64) { assetName = asset.Name.Substring(0, 64); - m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); + m_log.WarnFormat( + "[XASSET DB]: Name '{0}' for asset {1} truncated from {2} to {3} characters on add", + asset.Name, asset.ID, asset.Name.Length, assetName.Length); } string assetDescription = asset.Description; if (asset.Description.Length > 64) { assetDescription = asset.Description.Substring(0, 64); - m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); + m_log.WarnFormat( + "[XASSET DB]: Description '{0}' for asset {1} truncated from {2} to {3} characters on add", + asset.Description, asset.ID, asset.Description.Length, assetDescription.Length); } if (m_enableCompression) -- cgit v1.1 From e9f3cd1a60bfc5e936d1029495ada2c2bff99430 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 15 Mar 2013 23:17:54 +0000 Subject: Implement access time updates on assets for XAssetService. This only happens if access time is older than 30 days currently, in order to reduce database updates. The idea is to give some idea of assets which haven't been accessed for a very, very long time. These might conceivably be deleteable, though this will be a risk due to caching at other points in the chain. This is actually currently much less useable on the xasset service since access time is on metadata rather than the data itself. And many metadata entries may point to the same data. Probably need to address this. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 95 +++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 38 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index c2282c8..273fbca 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -50,6 +50,11 @@ namespace OpenSim.Data.MySQL get { return GetType().Assembly; } } + /// + /// Number of days that must pass before we update the access time on an asset when it has been fetched. + /// + private const int DaysBetweenAccessTimeUpdates = 30; + private bool m_enableCompression = false; private string m_connectionString; private object m_dbLock = new object(); @@ -133,7 +138,7 @@ namespace OpenSim.Data.MySQL dbcon.Open(); using (MySqlCommand cmd = new MySqlCommand( - "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", + "SELECT name, description, access_time, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", dbcon)) { cmd.Parameters.AddWithValue("?id", assetID.ToString()); @@ -171,12 +176,14 @@ namespace OpenSim.Data.MySQL // asset.ID, asset.Name, asset.Data.Length, compressedLength); } } + + UpdateAccessTime(asset.Metadata, (int)dbReader["access_time"]); } } } catch (Exception e) { - m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message); + m_log.Error("[MYSQL XASSET DATA]: Failure fetching asset " + assetID + ": " + e.Message); } } } @@ -303,41 +310,49 @@ namespace OpenSim.Data.MySQL } } -// private void UpdateAccessTime(AssetBase asset) -// { -// lock (m_dbLock) -// { -// using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) -// { -// dbcon.Open(); -// MySqlCommand cmd = -// new MySqlCommand("update assets set access_time=?access_time where id=?id", -// dbcon); -// -// // need to ensure we dispose -// try -// { -// using (cmd) -// { -// // create unix epoch time -// int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); -// cmd.Parameters.AddWithValue("?id", asset.ID); -// cmd.Parameters.AddWithValue("?access_time", now); -// cmd.ExecuteNonQuery(); -// cmd.Dispose(); -// } -// } -// catch (Exception e) -// { -// m_log.ErrorFormat( -// "[ASSETS DB]: " + -// "MySql failure updating access_time for asset {0} with name {1}" + Environment.NewLine + e.ToString() -// + Environment.NewLine + "Attempting reconnection", asset.FullID, asset.Name); -// } -// } -// } -// -// } + /// + /// Updates the access time of the asset if it was accessed above a given threshhold amount of time. + /// + /// + /// This gives us some insight into assets which haven't ben accessed for a long period. This is only done + /// over the threshold time to avoid excessive database writes as assets are fetched. + /// + /// + /// + private void UpdateAccessTime(AssetMetadata assetMetadata, int accessTime) + { + DateTime now = DateTime.UtcNow; + + if ((now - Utils.UnixTimeToDateTime(accessTime)).TotalDays < DaysBetweenAccessTimeUpdates) + return; + + lock (m_dbLock) + { + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + MySqlCommand cmd = + new MySqlCommand("update assets set access_time=?access_time where id=?id", dbcon); + + try + { + using (cmd) + { + // create unix epoch time + cmd.Parameters.AddWithValue("?id", assetMetadata.ID); + cmd.Parameters.AddWithValue("?access_time", (int)Utils.DateTimeToUnixTime(now)); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + m_log.ErrorFormat( + "[XASSET MYSQL DB]: Failure updating access_time for asset {0} with name {1}", + assetMetadata.ID, assetMetadata.Name); + } + } + } + } /// /// We assume we already have the m_dbLock. @@ -422,6 +437,8 @@ namespace OpenSim.Data.MySQL return assetExists; } + + /// /// Returns a list of AssetMetadata objects. The list is a subset of /// the entire data set offset by containing @@ -439,7 +456,7 @@ namespace OpenSim.Data.MySQL using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - MySqlCommand cmd = new MySqlCommand("SELECT name,description,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon); + MySqlCommand cmd = new MySqlCommand("SELECT name,description,access_time,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon); cmd.Parameters.AddWithValue("?start", start); cmd.Parameters.AddWithValue("?count", count); @@ -461,6 +478,8 @@ namespace OpenSim.Data.MySQL // We'll ignore this for now - it appears unused! // metadata.SHA1 = dbReader["hash"]); + UpdateAccessTime(metadata, (int)dbReader["access_time"]); + retList.Add(metadata); } } -- cgit v1.1 From 35843e8ec8c58803664ab33ad52fd8e64a8765c6 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 15 Mar 2013 23:42:16 +0000 Subject: Change the table and field names of XAssetService mysql db tables to be capitalized like Avatars, Friends, etc. Also fixes access time being set on assets rather than XAssetsMeta This is to try and be somewhat consistent with other service tables that are mainly in this style. No migration is supplied, since nobody should be using this service yet except on a test basis. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 92 +++++++++++++++++------------------ 1 file changed, 45 insertions(+), 47 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 273fbca..8c93825 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -138,10 +138,10 @@ namespace OpenSim.Data.MySQL dbcon.Open(); using (MySqlCommand cmd = new MySqlCommand( - "SELECT name, description, access_time, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", + "SELECT Name, Description, AccessTime, AssetType, Local, Temporary, AssetFlags, CreatorID, Data FROM XAssetsMeta JOIN XAssetsData ON XAssetsMeta.Hash = XAssetsData.Hash WHERE ID=?ID", dbcon)) { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); + cmd.Parameters.AddWithValue("?ID", assetID.ToString()); try { @@ -149,18 +149,18 @@ namespace OpenSim.Data.MySQL { if (dbReader.Read()) { - asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString()); - asset.Data = (byte[])dbReader["data"]; - asset.Description = (string)dbReader["description"]; + asset = new AssetBase(assetID, (string)dbReader["Name"], (sbyte)dbReader["AssetType"], dbReader["CreatorID"].ToString()); + asset.Data = (byte[])dbReader["Data"]; + asset.Description = (string)dbReader["Description"]; - string local = dbReader["local"].ToString(); + string local = dbReader["Local"].ToString(); if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) asset.Local = true; else asset.Local = false; - asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); - asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + asset.Temporary = Convert.ToBoolean(dbReader["Temporary"]); + asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); if (m_enableCompression) { @@ -177,13 +177,13 @@ namespace OpenSim.Data.MySQL } } - UpdateAccessTime(asset.Metadata, (int)dbReader["access_time"]); + UpdateAccessTime(asset.Metadata, (int)dbReader["AccessTime"]); } } } catch (Exception e) { - m_log.Error("[MYSQL XASSET DATA]: Failure fetching asset " + assetID + ": " + e.Message); + m_log.Error(string.Format("[MYSQL XASSET DATA]: Failure fetching asset {0}", assetID), e); } } } @@ -249,23 +249,23 @@ namespace OpenSim.Data.MySQL { using (MySqlCommand cmd = new MySqlCommand( - "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + - "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", + "replace INTO XAssetsMeta(ID, Hash, Name, Description, AssetType, Local, Temporary, CreateTime, AccessTime, AssetFlags, CreatorID)" + + "VALUES(?ID, ?Hash, ?Name, ?Description, ?AssetType, ?Local, ?Temporary, ?CreateTime, ?AccessTime, ?AssetFlags, ?CreatorID)", dbcon)) { // create unix epoch time int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - cmd.Parameters.AddWithValue("?id", asset.ID); - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?name", assetName); - cmd.Parameters.AddWithValue("?description", assetDescription); - cmd.Parameters.AddWithValue("?asset_type", asset.Type); - cmd.Parameters.AddWithValue("?local", asset.Local); - cmd.Parameters.AddWithValue("?temporary", asset.Temporary); - cmd.Parameters.AddWithValue("?create_time", now); - cmd.Parameters.AddWithValue("?access_time", now); - cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); - cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?ID", asset.ID); + cmd.Parameters.AddWithValue("?Hash", hash); + cmd.Parameters.AddWithValue("?Name", assetName); + cmd.Parameters.AddWithValue("?Description", assetDescription); + cmd.Parameters.AddWithValue("?AssetType", asset.Type); + cmd.Parameters.AddWithValue("?Local", asset.Local); + cmd.Parameters.AddWithValue("?Temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?CreateTime", now); + cmd.Parameters.AddWithValue("?AccessTime", now); + cmd.Parameters.AddWithValue("?CreatorID", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?AssetFlags", (int)asset.Flags); cmd.ExecuteNonQuery(); } } @@ -285,11 +285,11 @@ namespace OpenSim.Data.MySQL { using (MySqlCommand cmd = new MySqlCommand( - "INSERT INTO xassetsdata(hash, data) VALUES(?hash, ?data)", + "INSERT INTO XAssetsData(Hash, Data) VALUES(?Hash, ?Data)", dbcon)) { - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); + cmd.Parameters.AddWithValue("?Hash", hash); + cmd.Parameters.AddWithValue("?Data", asset.Data); cmd.ExecuteNonQuery(); } } @@ -332,15 +332,15 @@ namespace OpenSim.Data.MySQL { dbcon.Open(); MySqlCommand cmd = - new MySqlCommand("update assets set access_time=?access_time where id=?id", dbcon); + new MySqlCommand("update XAssetsMeta set AccessTime=?AccessTime where ID=?ID", dbcon); try { using (cmd) { // create unix epoch time - cmd.Parameters.AddWithValue("?id", assetMetadata.ID); - cmd.Parameters.AddWithValue("?access_time", (int)Utils.DateTimeToUnixTime(now)); + cmd.Parameters.AddWithValue("?ID", assetMetadata.ID); + cmd.Parameters.AddWithValue("?AccessTime", (int)Utils.DateTimeToUnixTime(now)); cmd.ExecuteNonQuery(); } } @@ -368,9 +368,9 @@ namespace OpenSim.Data.MySQL bool exists = false; - using (MySqlCommand cmd = new MySqlCommand("SELECT hash FROM xassetsdata WHERE hash=?hash", dbcon)) + using (MySqlCommand cmd = new MySqlCommand("SELECT Hash FROM XAssetsData WHERE Hash=?Hash", dbcon)) { - cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?Hash", hash); try { @@ -410,9 +410,9 @@ namespace OpenSim.Data.MySQL using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM xassetsmeta WHERE id=?id", dbcon)) + using (MySqlCommand cmd = new MySqlCommand("SELECT ID FROM XAssetsMeta WHERE ID=?ID", dbcon)) { - cmd.Parameters.AddWithValue("?id", uuid.ToString()); + cmd.Parameters.AddWithValue("?ID", uuid.ToString()); try { @@ -427,8 +427,7 @@ namespace OpenSim.Data.MySQL } catch (Exception e) { - m_log.ErrorFormat( - "[XASSETS DB]: MySql failure fetching asset {0}" + Environment.NewLine + e.ToString(), uuid); + m_log.Error(string.Format("[XASSETS DB]: MySql failure fetching asset {0}", uuid), e); } } } @@ -438,7 +437,6 @@ namespace OpenSim.Data.MySQL } - /// /// Returns a list of AssetMetadata objects. The list is a subset of /// the entire data set offset by containing @@ -456,7 +454,7 @@ namespace OpenSim.Data.MySQL using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - MySqlCommand cmd = new MySqlCommand("SELECT name,description,access_time,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon); + MySqlCommand cmd = new MySqlCommand("SELECT Name, Description, AccessTime, AssetType, Temporary, ID, AssetFlags, CreatorID FROM XAssetsMeta LIMIT ?start, ?count", dbcon); cmd.Parameters.AddWithValue("?start", start); cmd.Parameters.AddWithValue("?count", count); @@ -467,18 +465,18 @@ namespace OpenSim.Data.MySQL while (dbReader.Read()) { AssetMetadata metadata = new AssetMetadata(); - metadata.Name = (string)dbReader["name"]; - metadata.Description = (string)dbReader["description"]; - metadata.Type = (sbyte)dbReader["asset_type"]; - metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); // Not sure if this is correct. - metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); - metadata.FullID = DBGuid.FromDB(dbReader["id"]); - metadata.CreatorID = dbReader["creator_id"].ToString(); + metadata.Name = (string)dbReader["Name"]; + metadata.Description = (string)dbReader["Description"]; + metadata.Type = (sbyte)dbReader["AssetType"]; + metadata.Temporary = Convert.ToBoolean(dbReader["Temporary"]); // Not sure if this is correct. + metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); + metadata.FullID = DBGuid.FromDB(dbReader["ID"]); + metadata.CreatorID = dbReader["CreatorID"].ToString(); // We'll ignore this for now - it appears unused! // metadata.SHA1 = dbReader["hash"]); - UpdateAccessTime(metadata, (int)dbReader["access_time"]); + UpdateAccessTime(metadata, (int)dbReader["AccessTime"]); retList.Add(metadata); } @@ -504,9 +502,9 @@ namespace OpenSim.Data.MySQL { dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon)) + using (MySqlCommand cmd = new MySqlCommand("delete from XAssetsMeta where ID=?ID", dbcon)) { - cmd.Parameters.AddWithValue("?id", id); + cmd.Parameters.AddWithValue("?ID", id); cmd.ExecuteNonQuery(); } -- cgit v1.1 From dc0455e217b8da3fc8bd49e959b57d6021312b77 Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 28 Jun 2013 19:11:44 +0100 Subject: In XAssetService, on a delete asset request also delete the asset in any chained service. This eliminates the async migration since it causes a race condition with the "delete asset" console command --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 8c93825..91389ce 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -199,6 +199,8 @@ namespace OpenSim.Data.MySQL /// On failure : Throw an exception and attempt to reconnect to database public void StoreAsset(AssetBase asset) { +// m_log.DebugFormat("[XASSETS DB]: Storing asset {0} {1}", asset.Name, asset.ID); + lock (m_dbLock) { using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) -- cgit v1.1 From d1c3f8eef58b29eb8760eeb1ac03852a2387f927 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Mon, 31 Mar 2014 11:53:12 +0300 Subject: Added assets service method AssetsExist(), which returns whether the given list of assets exist. This method is used to optimize sending assets with embedded assets: e.g., when a Hypergrid visitor takes an item into the inventory. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 40 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 21 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 91389ce..1bf6a9a 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -397,45 +397,43 @@ namespace OpenSim.Data.MySQL } /// - /// Check if the asset exists in the database + /// Check if the assets exist in the database. /// - /// The asset UUID - /// true if it exists, false otherwise. - public bool ExistsAsset(UUID uuid) + /// The asset UUID's + /// For each asset: true if it exists, false otherwise + public bool[] AssetsExist(UUID[] uuids) { -// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + if (uuids.Length == 0) + return new bool[0]; + + HashSet exists = new HashSet(); - bool assetExists = false; + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT ID FROM assets WHERE ID IN ({0})", ids); lock (m_dbLock) { using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand("SELECT ID FROM XAssetsMeta WHERE ID=?ID", dbcon)) + using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) { - cmd.Parameters.AddWithValue("?ID", uuid.ToString()); - - try + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + while (dbReader.Read()) { - if (dbReader.Read()) - { -// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); - assetExists = true; - } + UUID id = DBGuid.FromDB(dbReader["ID"]); + exists.Add(id); } } - catch (Exception e) - { - m_log.Error(string.Format("[XASSETS DB]: MySql failure fetching asset {0}", uuid), e); - } } } } - return assetExists; + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exists.Contains(uuids[i]); + return results; } -- cgit v1.1 From 93a9ed2a6d5f6005ae213e2e91f1c79e2fd7f775 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 22 Apr 2014 12:04:06 +0300 Subject: Changed the maximum asset name and description lengths to constants. Also, pre-truncate the description of dynamic textures. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 1bf6a9a..430bb9f 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -210,18 +210,18 @@ namespace OpenSim.Data.MySQL using (MySqlTransaction transaction = dbcon.BeginTransaction()) { string assetName = asset.Name; - if (asset.Name.Length > 64) + if (asset.Name.Length > AssetBase.MAX_ASSET_NAME) { - assetName = asset.Name.Substring(0, 64); + assetName = asset.Name.Substring(0, AssetBase.MAX_ASSET_NAME); m_log.WarnFormat( "[XASSET DB]: Name '{0}' for asset {1} truncated from {2} to {3} characters on add", asset.Name, asset.ID, asset.Name.Length, assetName.Length); } string assetDescription = asset.Description; - if (asset.Description.Length > 64) + if (asset.Description.Length > AssetBase.MAX_ASSET_DESC) { - assetDescription = asset.Description.Substring(0, 64); + assetDescription = asset.Description.Substring(0, AssetBase.MAX_ASSET_DESC); m_log.WarnFormat( "[XASSET DB]: Description '{0}' for asset {1} truncated from {2} to {3} characters on add", asset.Description, asset.ID, asset.Description.Length, assetDescription.Length); -- cgit v1.1 From 998d7009a65def0a4debc9369d35b63611db5b55 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Tue, 22 Apr 2014 20:04:12 +0300 Subject: Eliminated many warnings --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 430bb9f..8361da2 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -346,7 +346,7 @@ namespace OpenSim.Data.MySQL cmd.ExecuteNonQuery(); } } - catch (Exception e) + catch (Exception) { m_log.ErrorFormat( "[XASSET MYSQL DB]: Failure updating access_time for asset {0} with name {1}", -- cgit v1.1 From fabab7414f346c91f5aa9a3ae3f29c262d131c6d Mon Sep 17 00:00:00 2001 From: Justin Clark-Casey (justincc) Date: Fri, 22 Aug 2014 20:34:33 +0100 Subject: Remove database connection locking in MySQLXAssetData. This is unnecessary as connections aren't shared and transactions are already in place where necessary. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 354 ++++++++++++++++------------------ 1 file changed, 168 insertions(+), 186 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 8361da2..af7e876 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -57,7 +57,6 @@ namespace OpenSim.Data.MySQL private bool m_enableCompression = false; private string m_connectionString; - private object m_dbLock = new object(); /// /// We can reuse this for all hashing since all methods are single-threaded through m_dbBLock @@ -131,60 +130,58 @@ namespace OpenSim.Data.MySQL // m_log.DebugFormat("[MYSQL XASSET DATA]: Looking for asset {0}", assetID); AssetBase asset = null; - lock (m_dbLock) + + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + dbcon.Open(); + + using (MySqlCommand cmd = new MySqlCommand( + "SELECT Name, Description, AccessTime, AssetType, Local, Temporary, AssetFlags, CreatorID, Data FROM XAssetsMeta JOIN XAssetsData ON XAssetsMeta.Hash = XAssetsData.Hash WHERE ID=?ID", + dbcon)) { - dbcon.Open(); + cmd.Parameters.AddWithValue("?ID", assetID.ToString()); - using (MySqlCommand cmd = new MySqlCommand( - "SELECT Name, Description, AccessTime, AssetType, Local, Temporary, AssetFlags, CreatorID, Data FROM XAssetsMeta JOIN XAssetsData ON XAssetsMeta.Hash = XAssetsData.Hash WHERE ID=?ID", - dbcon)) + try { - cmd.Parameters.AddWithValue("?ID", assetID.ToString()); - - try + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + if (dbReader.Read()) { - if (dbReader.Read()) - { - asset = new AssetBase(assetID, (string)dbReader["Name"], (sbyte)dbReader["AssetType"], dbReader["CreatorID"].ToString()); - asset.Data = (byte[])dbReader["Data"]; - asset.Description = (string)dbReader["Description"]; + asset = new AssetBase(assetID, (string)dbReader["Name"], (sbyte)dbReader["AssetType"], dbReader["CreatorID"].ToString()); + asset.Data = (byte[])dbReader["Data"]; + asset.Description = (string)dbReader["Description"]; - string local = dbReader["Local"].ToString(); - if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) - asset.Local = true; - else - asset.Local = false; + string local = dbReader["Local"].ToString(); + if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) + asset.Local = true; + else + asset.Local = false; - asset.Temporary = Convert.ToBoolean(dbReader["Temporary"]); - asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); + asset.Temporary = Convert.ToBoolean(dbReader["Temporary"]); + asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); - if (m_enableCompression) + if (m_enableCompression) + { + using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) { - using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) - { - MemoryStream outputStream = new MemoryStream(); - WebUtil.CopyStream(decompressionStream, outputStream, int.MaxValue); - // int compressedLength = asset.Data.Length; - asset.Data = outputStream.ToArray(); - - // m_log.DebugFormat( - // "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", - // asset.ID, asset.Name, asset.Data.Length, compressedLength); - } + MemoryStream outputStream = new MemoryStream(); + WebUtil.CopyStream(decompressionStream, outputStream, int.MaxValue); +// int compressedLength = asset.Data.Length; + asset.Data = outputStream.ToArray(); + +// m_log.DebugFormat( +// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", +// asset.ID, asset.Name, asset.Data.Length, compressedLength); } - - UpdateAccessTime(asset.Metadata, (int)dbReader["AccessTime"]); } + + UpdateAccessTime(asset.Metadata, (int)dbReader["AccessTime"]); } } - catch (Exception e) - { - m_log.Error(string.Format("[MYSQL XASSET DATA]: Failure fetching asset {0}", assetID), e); - } + } + catch (Exception e) + { + m_log.Error(string.Format("[MYSQL XASSET DATA]: Failure fetching asset {0}", assetID), e); } } } @@ -201,113 +198,110 @@ namespace OpenSim.Data.MySQL { // m_log.DebugFormat("[XASSETS DB]: Storing asset {0} {1}", asset.Name, asset.ID); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + dbcon.Open(); + + using (MySqlTransaction transaction = dbcon.BeginTransaction()) { - dbcon.Open(); + string assetName = asset.Name; + if (asset.Name.Length > AssetBase.MAX_ASSET_NAME) + { + assetName = asset.Name.Substring(0, AssetBase.MAX_ASSET_NAME); + m_log.WarnFormat( + "[XASSET DB]: Name '{0}' for asset {1} truncated from {2} to {3} characters on add", + asset.Name, asset.ID, asset.Name.Length, assetName.Length); + } - using (MySqlTransaction transaction = dbcon.BeginTransaction()) + string assetDescription = asset.Description; + if (asset.Description.Length > AssetBase.MAX_ASSET_DESC) { - string assetName = asset.Name; - if (asset.Name.Length > AssetBase.MAX_ASSET_NAME) - { - assetName = asset.Name.Substring(0, AssetBase.MAX_ASSET_NAME); - m_log.WarnFormat( - "[XASSET DB]: Name '{0}' for asset {1} truncated from {2} to {3} characters on add", - asset.Name, asset.ID, asset.Name.Length, assetName.Length); - } - - string assetDescription = asset.Description; - if (asset.Description.Length > AssetBase.MAX_ASSET_DESC) - { - assetDescription = asset.Description.Substring(0, AssetBase.MAX_ASSET_DESC); - m_log.WarnFormat( - "[XASSET DB]: Description '{0}' for asset {1} truncated from {2} to {3} characters on add", - asset.Description, asset.ID, asset.Description.Length, assetDescription.Length); - } + assetDescription = asset.Description.Substring(0, AssetBase.MAX_ASSET_DESC); + m_log.WarnFormat( + "[XASSET DB]: Description '{0}' for asset {1} truncated from {2} to {3} characters on add", + asset.Description, asset.ID, asset.Description.Length, assetDescription.Length); + } - if (m_enableCompression) - { - MemoryStream outputStream = new MemoryStream(); + if (m_enableCompression) + { + MemoryStream outputStream = new MemoryStream(); - using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) - { - // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); - // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. - compressionStream.Close(); - byte[] compressedData = outputStream.ToArray(); - asset.Data = compressedData; - } + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + { +// Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); + // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. + compressionStream.Close(); + byte[] compressedData = outputStream.ToArray(); + asset.Data = compressedData; } + } - byte[] hash = hasher.ComputeHash(asset.Data); + byte[] hash = hasher.ComputeHash(asset.Data); // m_log.DebugFormat( // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", // asset.ID, asset.Name, hash, compressedData.Length); + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO XAssetsMeta(ID, Hash, Name, Description, AssetType, Local, Temporary, CreateTime, AccessTime, AssetFlags, CreatorID)" + + "VALUES(?ID, ?Hash, ?Name, ?Description, ?AssetType, ?Local, ?Temporary, ?CreateTime, ?AccessTime, ?AssetFlags, ?CreatorID)", + dbcon)) + { + // create unix epoch time + int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); + cmd.Parameters.AddWithValue("?ID", asset.ID); + cmd.Parameters.AddWithValue("?Hash", hash); + cmd.Parameters.AddWithValue("?Name", assetName); + cmd.Parameters.AddWithValue("?Description", assetDescription); + cmd.Parameters.AddWithValue("?AssetType", asset.Type); + cmd.Parameters.AddWithValue("?Local", asset.Local); + cmd.Parameters.AddWithValue("?Temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?CreateTime", now); + cmd.Parameters.AddWithValue("?AccessTime", now); + cmd.Parameters.AddWithValue("?CreatorID", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?AssetFlags", (int)asset.Flags); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + + transaction.Rollback(); + + return; + } + + if (!ExistsData(dbcon, transaction, hash)) + { try { using (MySqlCommand cmd = new MySqlCommand( - "replace INTO XAssetsMeta(ID, Hash, Name, Description, AssetType, Local, Temporary, CreateTime, AccessTime, AssetFlags, CreatorID)" + - "VALUES(?ID, ?Hash, ?Name, ?Description, ?AssetType, ?Local, ?Temporary, ?CreateTime, ?AccessTime, ?AssetFlags, ?CreatorID)", + "INSERT INTO XAssetsData(Hash, Data) VALUES(?Hash, ?Data)", dbcon)) { - // create unix epoch time - int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - cmd.Parameters.AddWithValue("?ID", asset.ID); cmd.Parameters.AddWithValue("?Hash", hash); - cmd.Parameters.AddWithValue("?Name", assetName); - cmd.Parameters.AddWithValue("?Description", assetDescription); - cmd.Parameters.AddWithValue("?AssetType", asset.Type); - cmd.Parameters.AddWithValue("?Local", asset.Local); - cmd.Parameters.AddWithValue("?Temporary", asset.Temporary); - cmd.Parameters.AddWithValue("?CreateTime", now); - cmd.Parameters.AddWithValue("?AccessTime", now); - cmd.Parameters.AddWithValue("?CreatorID", asset.Metadata.CreatorID); - cmd.Parameters.AddWithValue("?AssetFlags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?Data", asset.Data); cmd.ExecuteNonQuery(); } } catch (Exception e) { - m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", asset.FullID, asset.Name, e.Message); transaction.Rollback(); return; } - - if (!ExistsData(dbcon, transaction, hash)) - { - try - { - using (MySqlCommand cmd = - new MySqlCommand( - "INSERT INTO XAssetsData(Hash, Data) VALUES(?Hash, ?Data)", - dbcon)) - { - cmd.Parameters.AddWithValue("?Hash", hash); - cmd.Parameters.AddWithValue("?Data", asset.Data); - cmd.ExecuteNonQuery(); - } - } - catch (Exception e) - { - m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); - - transaction.Rollback(); - - return; - } - } - - transaction.Commit(); } + + transaction.Commit(); } } } @@ -328,31 +322,28 @@ namespace OpenSim.Data.MySQL if ((now - Utils.UnixTimeToDateTime(accessTime)).TotalDays < DaysBetweenAccessTimeUpdates) return; - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - MySqlCommand cmd = - new MySqlCommand("update XAssetsMeta set AccessTime=?AccessTime where ID=?ID", dbcon); + dbcon.Open(); + MySqlCommand cmd = + new MySqlCommand("update XAssetsMeta set AccessTime=?AccessTime where ID=?ID", dbcon); - try - { - using (cmd) - { - // create unix epoch time - cmd.Parameters.AddWithValue("?ID", assetMetadata.ID); - cmd.Parameters.AddWithValue("?AccessTime", (int)Utils.DateTimeToUnixTime(now)); - cmd.ExecuteNonQuery(); - } - } - catch (Exception) + try + { + using (cmd) { - m_log.ErrorFormat( - "[XASSET MYSQL DB]: Failure updating access_time for asset {0} with name {1}", - assetMetadata.ID, assetMetadata.Name); + // create unix epoch time + cmd.Parameters.AddWithValue("?ID", assetMetadata.ID); + cmd.Parameters.AddWithValue("?AccessTime", (int)Utils.DateTimeToUnixTime(now)); + cmd.ExecuteNonQuery(); } } + catch (Exception) + { + m_log.ErrorFormat( + "[XASSET MYSQL DB]: Failure updating access_time for asset {0} with name {1}", + assetMetadata.ID, assetMetadata.Name); + } } } @@ -411,20 +402,17 @@ namespace OpenSim.Data.MySQL string ids = "'" + string.Join("','", uuids) + "'"; string sql = string.Format("SELECT ID FROM assets WHERE ID IN ({0})", ids); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + dbcon.Open(); + using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) { - dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - using (MySqlDataReader dbReader = cmd.ExecuteReader()) + while (dbReader.Read()) { - while (dbReader.Read()) - { - UUID id = DBGuid.FromDB(dbReader["ID"]); - exists.Add(id); - } + UUID id = DBGuid.FromDB(dbReader["ID"]); + exists.Add(id); } } } @@ -449,43 +437,40 @@ namespace OpenSim.Data.MySQL { List retList = new List(count); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - MySqlCommand cmd = new MySqlCommand("SELECT Name, Description, AccessTime, AssetType, Temporary, ID, AssetFlags, CreatorID FROM XAssetsMeta LIMIT ?start, ?count", dbcon); - cmd.Parameters.AddWithValue("?start", start); - cmd.Parameters.AddWithValue("?count", count); + dbcon.Open(); + MySqlCommand cmd = new MySqlCommand("SELECT Name, Description, AccessTime, AssetType, Temporary, ID, AssetFlags, CreatorID FROM XAssetsMeta LIMIT ?start, ?count", dbcon); + cmd.Parameters.AddWithValue("?start", start); + cmd.Parameters.AddWithValue("?count", count); - try + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - using (MySqlDataReader dbReader = cmd.ExecuteReader()) + while (dbReader.Read()) { - while (dbReader.Read()) - { - AssetMetadata metadata = new AssetMetadata(); - metadata.Name = (string)dbReader["Name"]; - metadata.Description = (string)dbReader["Description"]; - metadata.Type = (sbyte)dbReader["AssetType"]; - metadata.Temporary = Convert.ToBoolean(dbReader["Temporary"]); // Not sure if this is correct. - metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); - metadata.FullID = DBGuid.FromDB(dbReader["ID"]); - metadata.CreatorID = dbReader["CreatorID"].ToString(); - - // We'll ignore this for now - it appears unused! + AssetMetadata metadata = new AssetMetadata(); + metadata.Name = (string)dbReader["Name"]; + metadata.Description = (string)dbReader["Description"]; + metadata.Type = (sbyte)dbReader["AssetType"]; + metadata.Temporary = Convert.ToBoolean(dbReader["Temporary"]); // Not sure if this is correct. + metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); + metadata.FullID = DBGuid.FromDB(dbReader["ID"]); + metadata.CreatorID = dbReader["CreatorID"].ToString(); + + // We'll ignore this for now - it appears unused! // metadata.SHA1 = dbReader["hash"]); - UpdateAccessTime(metadata, (int)dbReader["AccessTime"]); + UpdateAccessTime(metadata, (int)dbReader["AccessTime"]); - retList.Add(metadata); - } + retList.Add(metadata); } } - catch (Exception e) - { - m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString()); - } + } + catch (Exception e) + { + m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString()); } } @@ -496,21 +481,18 @@ namespace OpenSim.Data.MySQL { // m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - using (MySqlCommand cmd = new MySqlCommand("delete from XAssetsMeta where ID=?ID", dbcon)) - { - cmd.Parameters.AddWithValue("?ID", id); - cmd.ExecuteNonQuery(); - } + dbcon.Open(); - // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we - // keep a reference count (?) + using (MySqlCommand cmd = new MySqlCommand("delete from XAssetsMeta where ID=?ID", dbcon)) + { + cmd.Parameters.AddWithValue("?ID", id); + cmd.ExecuteNonQuery(); } + + // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we + // keep a reference count (?) } return true; -- cgit v1.1