From 134f86e8d5c414409631b25b8c6f0ee45fbd8631 Mon Sep 17 00:00:00 2001 From: David Walter Seikel Date: Thu, 3 Nov 2016 21:44:39 +1000 Subject: Initial update to OpenSim 0.8.2.1 source code. --- OpenSim/Data/MySQL/MySQLXAssetData.cs | 431 +++++++++++++++++----------------- 1 file changed, 217 insertions(+), 214 deletions(-) (limited to 'OpenSim/Data/MySQL/MySQLXAssetData.cs') diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index e6ac22e..af7e876 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -50,9 +50,13 @@ namespace OpenSim.Data.MySQL get { return GetType().Assembly; } } + /// + /// Number of days that must pass before we update the access time on an asset when it has been fetched. + /// + private const int DaysBetweenAccessTimeUpdates = 30; + private bool m_enableCompression = false; private string m_connectionString; - private object m_dbLock = new object(); /// /// We can reuse this for all hashing since all methods are single-threaded through m_dbBLock @@ -126,58 +130,58 @@ namespace OpenSim.Data.MySQL // m_log.DebugFormat("[MYSQL XASSET DATA]: Looking for asset {0}", assetID); AssetBase asset = null; - lock (m_dbLock) + + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + dbcon.Open(); + + using (MySqlCommand cmd = new MySqlCommand( + "SELECT Name, Description, AccessTime, AssetType, Local, Temporary, AssetFlags, CreatorID, Data FROM XAssetsMeta JOIN XAssetsData ON XAssetsMeta.Hash = XAssetsData.Hash WHERE ID=?ID", + dbcon)) { - dbcon.Open(); + cmd.Parameters.AddWithValue("?ID", assetID.ToString()); - using (MySqlCommand cmd = new MySqlCommand( - "SELECT name, description, asset_type, local, temporary, asset_flags, creator_id, data FROM xassetsmeta JOIN xassetsdata ON xassetsmeta.hash = xassetsdata.hash WHERE id=?id", - dbcon)) + try { - cmd.Parameters.AddWithValue("?id", assetID.ToString()); - - try + using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + if (dbReader.Read()) { - if (dbReader.Read()) - { - asset = new AssetBase(assetID, (string)dbReader["name"], (sbyte)dbReader["asset_type"], dbReader["creator_id"].ToString()); - asset.Data = (byte[])dbReader["data"]; - asset.Description = (string)dbReader["description"]; + asset = new AssetBase(assetID, (string)dbReader["Name"], (sbyte)dbReader["AssetType"], dbReader["CreatorID"].ToString()); + asset.Data = (byte[])dbReader["Data"]; + asset.Description = (string)dbReader["Description"]; - string local = dbReader["local"].ToString(); - if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) - asset.Local = true; - else - asset.Local = false; + string local = dbReader["Local"].ToString(); + if (local.Equals("1") || local.Equals("true", StringComparison.InvariantCultureIgnoreCase)) + asset.Local = true; + else + asset.Local = false; - asset.Temporary = Convert.ToBoolean(dbReader["temporary"]); - asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); + asset.Temporary = Convert.ToBoolean(dbReader["Temporary"]); + asset.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); - if (m_enableCompression) + if (m_enableCompression) + { + using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) { - using (GZipStream decompressionStream = new GZipStream(new MemoryStream(asset.Data), CompressionMode.Decompress)) - { - MemoryStream outputStream = new MemoryStream(); - WebUtil.CopyStream(decompressionStream, outputStream, int.MaxValue); - // int compressedLength = asset.Data.Length; - asset.Data = outputStream.ToArray(); - - // m_log.DebugFormat( - // "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", - // asset.ID, asset.Name, asset.Data.Length, compressedLength); - } + MemoryStream outputStream = new MemoryStream(); + WebUtil.CopyStream(decompressionStream, outputStream, int.MaxValue); +// int compressedLength = asset.Data.Length; + asset.Data = outputStream.ToArray(); + +// m_log.DebugFormat( +// "[XASSET DB]: Decompressed {0} {1} to {2} bytes from {3}", +// asset.ID, asset.Name, asset.Data.Length, compressedLength); } } + + UpdateAccessTime(asset.Metadata, (int)dbReader["AccessTime"]); } } - catch (Exception e) - { - m_log.Error("[MYSQL XASSET DATA]: MySql failure fetching asset " + assetID + ": " + e.Message); - } + } + catch (Exception e) + { + m_log.Error(string.Format("[MYSQL XASSET DATA]: Failure fetching asset {0}", assetID), e); } } } @@ -192,148 +196,156 @@ namespace OpenSim.Data.MySQL /// On failure : Throw an exception and attempt to reconnect to database public void StoreAsset(AssetBase asset) { - lock (m_dbLock) +// m_log.DebugFormat("[XASSETS DB]: Storing asset {0} {1}", asset.Name, asset.ID); + + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + dbcon.Open(); + + using (MySqlTransaction transaction = dbcon.BeginTransaction()) { - dbcon.Open(); + string assetName = asset.Name; + if (asset.Name.Length > AssetBase.MAX_ASSET_NAME) + { + assetName = asset.Name.Substring(0, AssetBase.MAX_ASSET_NAME); + m_log.WarnFormat( + "[XASSET DB]: Name '{0}' for asset {1} truncated from {2} to {3} characters on add", + asset.Name, asset.ID, asset.Name.Length, assetName.Length); + } - using (MySqlTransaction transaction = dbcon.BeginTransaction()) + string assetDescription = asset.Description; + if (asset.Description.Length > AssetBase.MAX_ASSET_DESC) { - string assetName = asset.Name; - if (asset.Name.Length > 64) - { - assetName = asset.Name.Substring(0, 64); - m_log.Warn("[XASSET DB]: Name field truncated from " + asset.Name.Length + " to " + assetName.Length + " characters on add"); - } - - string assetDescription = asset.Description; - if (asset.Description.Length > 64) - { - assetDescription = asset.Description.Substring(0, 64); - m_log.Warn("[XASSET DB]: Description field truncated from " + asset.Description.Length + " to " + assetDescription.Length + " characters on add"); - } + assetDescription = asset.Description.Substring(0, AssetBase.MAX_ASSET_DESC); + m_log.WarnFormat( + "[XASSET DB]: Description '{0}' for asset {1} truncated from {2} to {3} characters on add", + asset.Description, asset.ID, asset.Description.Length, assetDescription.Length); + } - if (m_enableCompression) - { - MemoryStream outputStream = new MemoryStream(); + if (m_enableCompression) + { + MemoryStream outputStream = new MemoryStream(); - using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) - { - // Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); - // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. - compressionStream.Close(); - byte[] compressedData = outputStream.ToArray(); - asset.Data = compressedData; - } + using (GZipStream compressionStream = new GZipStream(outputStream, CompressionMode.Compress, false)) + { +// Console.WriteLine(WebUtil.CopyTo(new MemoryStream(asset.Data), compressionStream, int.MaxValue)); + // We have to close the compression stream in order to make sure it writes everything out to the underlying memory output stream. + compressionStream.Close(); + byte[] compressedData = outputStream.ToArray(); + asset.Data = compressedData; } + } - byte[] hash = hasher.ComputeHash(asset.Data); + byte[] hash = hasher.ComputeHash(asset.Data); // m_log.DebugFormat( // "[XASSET DB]: Compressed data size for {0} {1}, hash {2} is {3}", // asset.ID, asset.Name, hash, compressedData.Length); + try + { + using (MySqlCommand cmd = + new MySqlCommand( + "replace INTO XAssetsMeta(ID, Hash, Name, Description, AssetType, Local, Temporary, CreateTime, AccessTime, AssetFlags, CreatorID)" + + "VALUES(?ID, ?Hash, ?Name, ?Description, ?AssetType, ?Local, ?Temporary, ?CreateTime, ?AccessTime, ?AssetFlags, ?CreatorID)", + dbcon)) + { + // create unix epoch time + int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); + cmd.Parameters.AddWithValue("?ID", asset.ID); + cmd.Parameters.AddWithValue("?Hash", hash); + cmd.Parameters.AddWithValue("?Name", assetName); + cmd.Parameters.AddWithValue("?Description", assetDescription); + cmd.Parameters.AddWithValue("?AssetType", asset.Type); + cmd.Parameters.AddWithValue("?Local", asset.Local); + cmd.Parameters.AddWithValue("?Temporary", asset.Temporary); + cmd.Parameters.AddWithValue("?CreateTime", now); + cmd.Parameters.AddWithValue("?AccessTime", now); + cmd.Parameters.AddWithValue("?CreatorID", asset.Metadata.CreatorID); + cmd.Parameters.AddWithValue("?AssetFlags", (int)asset.Flags); + cmd.ExecuteNonQuery(); + } + } + catch (Exception e) + { + m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + asset.FullID, asset.Name, e.Message); + + transaction.Rollback(); + + return; + } + + if (!ExistsData(dbcon, transaction, hash)) + { try { using (MySqlCommand cmd = new MySqlCommand( - "replace INTO xassetsmeta(id, hash, name, description, asset_type, local, temporary, create_time, access_time, asset_flags, creator_id)" + - "VALUES(?id, ?hash, ?name, ?description, ?asset_type, ?local, ?temporary, ?create_time, ?access_time, ?asset_flags, ?creator_id)", + "INSERT INTO XAssetsData(Hash, Data) VALUES(?Hash, ?Data)", dbcon)) { - // create unix epoch time - int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); - cmd.Parameters.AddWithValue("?id", asset.ID); - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?name", assetName); - cmd.Parameters.AddWithValue("?description", assetDescription); - cmd.Parameters.AddWithValue("?asset_type", asset.Type); - cmd.Parameters.AddWithValue("?local", asset.Local); - cmd.Parameters.AddWithValue("?temporary", asset.Temporary); - cmd.Parameters.AddWithValue("?create_time", now); - cmd.Parameters.AddWithValue("?access_time", now); - cmd.Parameters.AddWithValue("?creator_id", asset.Metadata.CreatorID); - cmd.Parameters.AddWithValue("?asset_flags", (int)asset.Flags); + cmd.Parameters.AddWithValue("?Hash", hash); + cmd.Parameters.AddWithValue("?Data", asset.Data); cmd.ExecuteNonQuery(); } } catch (Exception e) { - m_log.ErrorFormat("[ASSET DB]: MySQL failure creating asset metadata {0} with name \"{1}\". Error: {2}", + m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", asset.FullID, asset.Name, e.Message); transaction.Rollback(); return; } - - if (!ExistsData(dbcon, transaction, hash)) - { - try - { - using (MySqlCommand cmd = - new MySqlCommand( - "INSERT INTO xassetsdata(hash, data) VALUES(?hash, ?data)", - dbcon)) - { - cmd.Parameters.AddWithValue("?hash", hash); - cmd.Parameters.AddWithValue("?data", asset.Data); - cmd.ExecuteNonQuery(); - } - } - catch (Exception e) - { - m_log.ErrorFormat("[XASSET DB]: MySQL failure creating asset data {0} with name \"{1}\". Error: {2}", - asset.FullID, asset.Name, e.Message); - - transaction.Rollback(); - - return; - } - } - - transaction.Commit(); } + + transaction.Commit(); } } } -// private void UpdateAccessTime(AssetBase asset) -// { -// lock (m_dbLock) -// { -// using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) -// { -// dbcon.Open(); -// MySqlCommand cmd = -// new MySqlCommand("update assets set access_time=?access_time where id=?id", -// dbcon); -// -// // need to ensure we dispose -// try -// { -// using (cmd) -// { -// // create unix epoch time -// int now = (int)Utils.DateTimeToUnixTime(DateTime.UtcNow); -// cmd.Parameters.AddWithValue("?id", asset.ID); -// cmd.Parameters.AddWithValue("?access_time", now); -// cmd.ExecuteNonQuery(); -// cmd.Dispose(); -// } -// } -// catch (Exception e) -// { -// m_log.ErrorFormat( -// "[ASSETS DB]: " + -// "MySql failure updating access_time for asset {0} with name {1}" + Environment.NewLine + e.ToString() -// + Environment.NewLine + "Attempting reconnection", asset.FullID, asset.Name); -// } -// } -// } -// -// } + /// + /// Updates the access time of the asset if it was accessed above a given threshhold amount of time. + /// + /// + /// This gives us some insight into assets which haven't ben accessed for a long period. This is only done + /// over the threshold time to avoid excessive database writes as assets are fetched. + /// + /// + /// + private void UpdateAccessTime(AssetMetadata assetMetadata, int accessTime) + { + DateTime now = DateTime.UtcNow; + + if ((now - Utils.UnixTimeToDateTime(accessTime)).TotalDays < DaysBetweenAccessTimeUpdates) + return; + + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + { + dbcon.Open(); + MySqlCommand cmd = + new MySqlCommand("update XAssetsMeta set AccessTime=?AccessTime where ID=?ID", dbcon); + + try + { + using (cmd) + { + // create unix epoch time + cmd.Parameters.AddWithValue("?ID", assetMetadata.ID); + cmd.Parameters.AddWithValue("?AccessTime", (int)Utils.DateTimeToUnixTime(now)); + cmd.ExecuteNonQuery(); + } + } + catch (Exception) + { + m_log.ErrorFormat( + "[XASSET MYSQL DB]: Failure updating access_time for asset {0} with name {1}", + assetMetadata.ID, assetMetadata.Name); + } + } + } /// /// We assume we already have the m_dbLock. @@ -349,9 +361,9 @@ namespace OpenSim.Data.MySQL bool exists = false; - using (MySqlCommand cmd = new MySqlCommand("SELECT hash FROM xassetsdata WHERE hash=?hash", dbcon)) + using (MySqlCommand cmd = new MySqlCommand("SELECT Hash FROM XAssetsData WHERE Hash=?Hash", dbcon)) { - cmd.Parameters.AddWithValue("?hash", hash); + cmd.Parameters.AddWithValue("?Hash", hash); try { @@ -376,48 +388,43 @@ namespace OpenSim.Data.MySQL } /// - /// Check if the asset exists in the database + /// Check if the assets exist in the database. /// - /// The asset UUID - /// true if it exists, false otherwise. - public bool ExistsAsset(UUID uuid) + /// The asset UUID's + /// For each asset: true if it exists, false otherwise + public bool[] AssetsExist(UUID[] uuids) { -// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + if (uuids.Length == 0) + return new bool[0]; + + HashSet exists = new HashSet(); - bool assetExists = false; + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT ID FROM assets WHERE ID IN ({0})", ids); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) + dbcon.Open(); + using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) { - dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM xassetsmeta WHERE id=?id", dbcon)) + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - cmd.Parameters.AddWithValue("?id", uuid.ToString()); - - try - { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) - { - if (dbReader.Read()) - { -// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); - assetExists = true; - } - } - } - catch (Exception e) + while (dbReader.Read()) { - m_log.ErrorFormat( - "[XASSETS DB]: MySql failure fetching asset {0}" + Environment.NewLine + e.ToString(), uuid); + UUID id = DBGuid.FromDB(dbReader["ID"]); + exists.Add(id); } } } } - return assetExists; + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exists.Contains(uuids[i]); + return results; } + /// /// Returns a list of AssetMetadata objects. The list is a subset of /// the entire data set offset by containing @@ -430,41 +437,40 @@ namespace OpenSim.Data.MySQL { List retList = new List(count); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - MySqlCommand cmd = new MySqlCommand("SELECT name,description,asset_type,temporary,id,asset_flags,creator_id FROM xassetsmeta LIMIT ?start, ?count", dbcon); - cmd.Parameters.AddWithValue("?start", start); - cmd.Parameters.AddWithValue("?count", count); + dbcon.Open(); + MySqlCommand cmd = new MySqlCommand("SELECT Name, Description, AccessTime, AssetType, Temporary, ID, AssetFlags, CreatorID FROM XAssetsMeta LIMIT ?start, ?count", dbcon); + cmd.Parameters.AddWithValue("?start", start); + cmd.Parameters.AddWithValue("?count", count); - try + try + { + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - using (MySqlDataReader dbReader = cmd.ExecuteReader()) + while (dbReader.Read()) { - while (dbReader.Read()) - { - AssetMetadata metadata = new AssetMetadata(); - metadata.Name = (string)dbReader["name"]; - metadata.Description = (string)dbReader["description"]; - metadata.Type = (sbyte)dbReader["asset_type"]; - metadata.Temporary = Convert.ToBoolean(dbReader["temporary"]); // Not sure if this is correct. - metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["asset_flags"]); - metadata.FullID = DBGuid.FromDB(dbReader["id"]); - metadata.CreatorID = dbReader["creator_id"].ToString(); - - // We'll ignore this for now - it appears unused! + AssetMetadata metadata = new AssetMetadata(); + metadata.Name = (string)dbReader["Name"]; + metadata.Description = (string)dbReader["Description"]; + metadata.Type = (sbyte)dbReader["AssetType"]; + metadata.Temporary = Convert.ToBoolean(dbReader["Temporary"]); // Not sure if this is correct. + metadata.Flags = (AssetFlags)Convert.ToInt32(dbReader["AssetFlags"]); + metadata.FullID = DBGuid.FromDB(dbReader["ID"]); + metadata.CreatorID = dbReader["CreatorID"].ToString(); + + // We'll ignore this for now - it appears unused! // metadata.SHA1 = dbReader["hash"]); - retList.Add(metadata); - } + UpdateAccessTime(metadata, (int)dbReader["AccessTime"]); + + retList.Add(metadata); } } - catch (Exception e) - { - m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString()); - } + } + catch (Exception e) + { + m_log.Error("[XASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString()); } } @@ -475,21 +481,18 @@ namespace OpenSim.Data.MySQL { // m_log.DebugFormat("[XASSETS DB]: Deleting asset {0}", id); - lock (m_dbLock) + using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { - using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) - { - dbcon.Open(); - - using (MySqlCommand cmd = new MySqlCommand("delete from xassetsmeta where id=?id", dbcon)) - { - cmd.Parameters.AddWithValue("?id", id); - cmd.ExecuteNonQuery(); - } + dbcon.Open(); - // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we - // keep a reference count (?) + using (MySqlCommand cmd = new MySqlCommand("delete from XAssetsMeta where ID=?ID", dbcon)) + { + cmd.Parameters.AddWithValue("?ID", id); + cmd.ExecuteNonQuery(); } + + // TODO: How do we deal with data from deleted assets? Probably not easily reapable unless we + // keep a reference count (?) } return true; -- cgit v1.1