diff options
author | AliciaRaven | 2015-07-12 01:35:57 +0100 |
---|---|---|
committer | Melanie Thielker | 2015-07-12 04:03:08 +0200 |
commit | f3f748ed111223bdf9cbc25fcabb18b4ab427d37 (patch) | |
tree | 382d6390ec5ae88149bd03df8d5d82787df60bda /OpenSim/Services/FSAssetService | |
parent | More cleanup, add missing refs to prebuild, remove refs to stuff not (diff) | |
download | opensim-SC_OLD-f3f748ed111223bdf9cbc25fcabb18b4ab427d37.zip opensim-SC_OLD-f3f748ed111223bdf9cbc25fcabb18b4ab427d37.tar.gz opensim-SC_OLD-f3f748ed111223bdf9cbc25fcabb18b4ab427d37.tar.bz2 opensim-SC_OLD-f3f748ed111223bdf9cbc25fcabb18b4ab427d37.tar.xz |
Bringing FSAssets more inline with current OpenSim standards. * If no connection string found in assets config, fallback to using default database config * Create database storage interface to allow other db connectors to be written at some point * Add MySQL migrations file to create the initial db table * Added new config option named DaysBetweenAccessTimeUpdates to reduce db load by only updating access times when fetching assets if the last access time was over the threshold. This idea was taken from XAssets service. * Change log message headers to indicate FS assets is the source not just assets
Signed-off-by: Melanie Thielker <melanie@t-data.com>
Diffstat (limited to 'OpenSim/Services/FSAssetService')
-rw-r--r-- | OpenSim/Services/FSAssetService/FSAssetService.cs | 80 |
1 files changed, 51 insertions, 29 deletions
diff --git a/OpenSim/Services/FSAssetService/FSAssetService.cs b/OpenSim/Services/FSAssetService/FSAssetService.cs index 3662e27..8276f33 100644 --- a/OpenSim/Services/FSAssetService/FSAssetService.cs +++ b/OpenSim/Services/FSAssetService/FSAssetService.cs | |||
@@ -33,6 +33,7 @@ using System.IO.Compression; | |||
33 | using System.Text; | 33 | using System.Text; |
34 | using System.Threading; | 34 | using System.Threading; |
35 | using System.Reflection; | 35 | using System.Reflection; |
36 | using OpenSim.Data; | ||
36 | using OpenSim.Framework; | 37 | using OpenSim.Framework; |
37 | using OpenSim.Framework.Console; | 38 | using OpenSim.Framework.Console; |
38 | using OpenSim.Server.Base; | 39 | using OpenSim.Server.Base; |
@@ -47,9 +48,7 @@ namespace OpenSim.Services.FSAssetService | |||
47 | { | 48 | { |
48 | public class FSAssetConnector : ServiceBase, IAssetService | 49 | public class FSAssetConnector : ServiceBase, IAssetService |
49 | { | 50 | { |
50 | private static readonly ILog m_log = | 51 | private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
51 | LogManager.GetLogger( | ||
52 | MethodBase.GetCurrentMethod().DeclaringType); | ||
53 | 52 | ||
54 | static System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); | 53 | static System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); |
55 | static SHA256CryptoServiceProvider SHA256 = new SHA256CryptoServiceProvider(); | 54 | static SHA256CryptoServiceProvider SHA256 = new SHA256CryptoServiceProvider(); |
@@ -64,9 +63,7 @@ namespace OpenSim.Services.FSAssetService | |||
64 | } | 63 | } |
65 | 64 | ||
66 | protected IAssetLoader m_AssetLoader = null; | 65 | protected IAssetLoader m_AssetLoader = null; |
67 | protected string m_ConnectionString; | 66 | protected IFSAssetDataPlugin m_DataConnector = null; |
68 | protected FSAssetConnectorData m_DataConnector = null; | ||
69 | protected string m_FsckProgram; | ||
70 | protected IAssetService m_FallbackService; | 67 | protected IAssetService m_FallbackService; |
71 | protected Thread m_WriterThread; | 68 | protected Thread m_WriterThread; |
72 | protected Thread m_StatsThread; | 69 | protected Thread m_StatsThread; |
@@ -78,7 +75,6 @@ namespace OpenSim.Services.FSAssetService | |||
78 | protected int m_missingAssets = 0; | 75 | protected int m_missingAssets = 0; |
79 | protected int m_missingAssetsFS = 0; | 76 | protected int m_missingAssetsFS = 0; |
80 | protected string m_FSBase; | 77 | protected string m_FSBase; |
81 | protected string m_Realm; | ||
82 | 78 | ||
83 | public FSAssetConnector(IConfigSource config) | 79 | public FSAssetConnector(IConfigSource config) |
84 | : this(config, "AssetService") | 80 | : this(config, "AssetService") |
@@ -87,8 +83,6 @@ namespace OpenSim.Services.FSAssetService | |||
87 | 83 | ||
88 | public FSAssetConnector(IConfigSource config, string configName) : base(config) | 84 | public FSAssetConnector(IConfigSource config, string configName) : base(config) |
89 | { | 85 | { |
90 | m_FsckProgram = string.Empty; | ||
91 | |||
92 | MainConsole.Instance.Commands.AddCommand("fs", false, | 86 | MainConsole.Instance.Commands.AddCommand("fs", false, |
93 | "show assets", "show assets", "Show asset stats", | 87 | "show assets", "show assets", "Show asset stats", |
94 | HandleShowAssets); | 88 | HandleShowAssets); |
@@ -109,35 +103,63 @@ namespace OpenSim.Services.FSAssetService | |||
109 | HandleImportAssets); | 103 | HandleImportAssets); |
110 | 104 | ||
111 | IConfig assetConfig = config.Configs[configName]; | 105 | IConfig assetConfig = config.Configs[configName]; |
106 | |||
112 | if (assetConfig == null) | 107 | if (assetConfig == null) |
113 | { | ||
114 | throw new Exception("No AssetService configuration"); | 108 | throw new Exception("No AssetService configuration"); |
115 | } | ||
116 | 109 | ||
117 | m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty); | 110 | // Get Database Connector from Asset Config (If present) |
118 | if (m_ConnectionString == string.Empty) | 111 | string dllName = assetConfig.GetString("StorageProvider", string.Empty); |
112 | string m_ConnectionString = assetConfig.GetString("ConnectionString", string.Empty); | ||
113 | string m_Realm = assetConfig.GetString("Realm", "fsassets"); | ||
114 | |||
115 | int SkipAccessTimeDays = assetConfig.GetInt("DaysBetweenAccessTimeUpdates", 0); | ||
116 | |||
117 | // If not found above, fallback to Database defaults | ||
118 | IConfig dbConfig = config.Configs["DatabaseService"]; | ||
119 | |||
120 | if (dbConfig != null) | ||
119 | { | 121 | { |
120 | throw new Exception("Missing database connection string"); | 122 | if (dllName == String.Empty) |
123 | dllName = dbConfig.GetString("StorageProvider", String.Empty); | ||
124 | |||
125 | if (m_ConnectionString == String.Empty) | ||
126 | m_ConnectionString = dbConfig.GetString("ConnectionString", String.Empty); | ||
121 | } | 127 | } |
122 | 128 | ||
123 | m_Realm = assetConfig.GetString("Realm", "fsassets"); | 129 | // No databse connection found in either config |
130 | if (dllName.Equals(String.Empty)) | ||
131 | throw new Exception("No StorageProvider configured"); | ||
132 | |||
133 | if (m_ConnectionString.Equals(String.Empty)) | ||
134 | throw new Exception("Missing database connection string"); | ||
135 | |||
136 | // Create Storage Provider | ||
137 | m_DataConnector = LoadPlugin<IFSAssetDataPlugin>(dllName); | ||
138 | |||
139 | if (m_DataConnector == null) | ||
140 | throw new Exception(string.Format("Could not find a storage interface in the module {0}", dllName)); | ||
141 | |||
142 | // Initialize DB And perform any migrations required | ||
143 | m_DataConnector.Initialise(m_ConnectionString, m_Realm, SkipAccessTimeDays); | ||
124 | 144 | ||
125 | m_DataConnector = new FSAssetConnectorData(m_ConnectionString, m_Realm); | 145 | // Setup Fallback Service |
126 | string str = assetConfig.GetString("FallbackService", string.Empty); | 146 | string str = assetConfig.GetString("FallbackService", string.Empty); |
147 | |||
127 | if (str != string.Empty) | 148 | if (str != string.Empty) |
128 | { | 149 | { |
129 | object[] args = new object[] { config }; | 150 | object[] args = new object[] { config }; |
130 | m_FallbackService = LoadPlugin<IAssetService>(str, args); | 151 | m_FallbackService = LoadPlugin<IAssetService>(str, args); |
131 | if (m_FallbackService != null) | 152 | if (m_FallbackService != null) |
132 | { | 153 | { |
133 | m_log.Info("[FALLBACK]: Fallback service loaded"); | 154 | m_log.Info("[FSASSETS]: Fallback service loaded"); |
134 | } | 155 | } |
135 | else | 156 | else |
136 | { | 157 | { |
137 | m_log.Error("[FALLBACK]: Failed to load fallback service"); | 158 | m_log.Error("[FSASSETS]: Failed to load fallback service"); |
138 | } | 159 | } |
139 | } | 160 | } |
140 | 161 | ||
162 | // Setup directory structure including temp directory | ||
141 | m_SpoolDirectory = assetConfig.GetString("SpoolDirectory", "/tmp"); | 163 | m_SpoolDirectory = assetConfig.GetString("SpoolDirectory", "/tmp"); |
142 | 164 | ||
143 | string spoolTmp = Path.Combine(m_SpoolDirectory, "spool"); | 165 | string spoolTmp = Path.Combine(m_SpoolDirectory, "spool"); |
@@ -147,7 +169,7 @@ namespace OpenSim.Services.FSAssetService | |||
147 | m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty); | 169 | m_FSBase = assetConfig.GetString("BaseDirectory", String.Empty); |
148 | if (m_FSBase == String.Empty) | 170 | if (m_FSBase == String.Empty) |
149 | { | 171 | { |
150 | m_log.ErrorFormat("[ASSET]: BaseDirectory not specified"); | 172 | m_log.ErrorFormat("[FSASSETS]: BaseDirectory not specified"); |
151 | throw new Exception("Configuration error"); | 173 | throw new Exception("Configuration error"); |
152 | } | 174 | } |
153 | 175 | ||
@@ -156,14 +178,14 @@ namespace OpenSim.Services.FSAssetService | |||
156 | { | 178 | { |
157 | m_AssetLoader = LoadPlugin<IAssetLoader>(loader); | 179 | m_AssetLoader = LoadPlugin<IAssetLoader>(loader); |
158 | string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty); | 180 | string loaderArgs = assetConfig.GetString("AssetLoaderArgs", string.Empty); |
159 | m_log.InfoFormat("[ASSET]: Loading default asset set from {0}", loaderArgs); | 181 | m_log.InfoFormat("[FSASSETS]: Loading default asset set from {0}", loaderArgs); |
160 | m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs, | 182 | m_AssetLoader.ForEachDefaultXmlAsset(loaderArgs, |
161 | delegate(AssetBase a) | 183 | delegate(AssetBase a) |
162 | { | 184 | { |
163 | Store(a, false); | 185 | Store(a, false); |
164 | }); | 186 | }); |
165 | } | 187 | } |
166 | m_log.Info("[ASSET]: FS asset service enabled"); | 188 | m_log.Info("[FSASSETS]: FS asset service enabled"); |
167 | 189 | ||
168 | m_WriterThread = new Thread(Writer); | 190 | m_WriterThread = new Thread(Writer); |
169 | m_WriterThread.Start(); | 191 | m_WriterThread.Start(); |
@@ -184,7 +206,7 @@ namespace OpenSim.Services.FSAssetService | |||
184 | double avg = (double)m_readTicks / (double)m_readCount; | 206 | double avg = (double)m_readTicks / (double)m_readCount; |
185 | // if (avg > 10000) | 207 | // if (avg > 10000) |
186 | // Environment.Exit(0); | 208 | // Environment.Exit(0); |
187 | m_log.InfoFormat("[ASSET]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS); | 209 | m_log.InfoFormat("[FSASSETS]: Read stats: {0} files, {1} ticks, avg {2:F2}, missing {3}, FS {4}", m_readCount, m_readTicks, (double)m_readTicks / (double)m_readCount, m_missingAssets, m_missingAssetsFS); |
188 | } | 210 | } |
189 | m_readCount = 0; | 211 | m_readCount = 0; |
190 | m_readTicks = 0; | 212 | m_readTicks = 0; |
@@ -196,7 +218,7 @@ namespace OpenSim.Services.FSAssetService | |||
196 | 218 | ||
197 | private void Writer() | 219 | private void Writer() |
198 | { | 220 | { |
199 | m_log.Info("[ASSET]: Writer started"); | 221 | m_log.Info("[FSASSETS]: Writer started"); |
200 | 222 | ||
201 | while (true) | 223 | while (true) |
202 | { | 224 | { |
@@ -236,7 +258,7 @@ namespace OpenSim.Services.FSAssetService | |||
236 | int totalTicks = System.Environment.TickCount - tickCount; | 258 | int totalTicks = System.Environment.TickCount - tickCount; |
237 | if (totalTicks > 0) // Wrap? | 259 | if (totalTicks > 0) // Wrap? |
238 | { | 260 | { |
239 | m_log.InfoFormat("[ASSET]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length); | 261 | m_log.InfoFormat("[FSASSETS]: Write cycle complete, {0} files, {1} ticks, avg {2:F2}", files.Length, totalTicks, (double)totalTicks / (double)files.Length); |
240 | } | 262 | } |
241 | } | 263 | } |
242 | 264 | ||
@@ -326,13 +348,13 @@ namespace OpenSim.Services.FSAssetService | |||
326 | asset.Metadata.ContentType = | 348 | asset.Metadata.ContentType = |
327 | SLUtil.SLAssetTypeToContentType((int)asset.Type); | 349 | SLUtil.SLAssetTypeToContentType((int)asset.Type); |
328 | sha = GetSHA256Hash(asset.Data); | 350 | sha = GetSHA256Hash(asset.Data); |
329 | m_log.InfoFormat("[FALLBACK]: Added asset {0} from fallback to local store", id); | 351 | m_log.InfoFormat("[FSASSETS]: Added asset {0} from fallback to local store", id); |
330 | Store(asset); | 352 | Store(asset); |
331 | } | 353 | } |
332 | } | 354 | } |
333 | if (asset == null) | 355 | if (asset == null) |
334 | { | 356 | { |
335 | // m_log.InfoFormat("[ASSET]: Asset {0} not found", id); | 357 | // m_log.InfoFormat("[FSASSETS]: Asset {0} not found", id); |
336 | m_missingAssets++; | 358 | m_missingAssets++; |
337 | } | 359 | } |
338 | return asset; | 360 | return asset; |
@@ -353,13 +375,13 @@ namespace OpenSim.Services.FSAssetService | |||
353 | asset.Metadata.ContentType = | 375 | asset.Metadata.ContentType = |
354 | SLUtil.SLAssetTypeToContentType((int)asset.Type); | 376 | SLUtil.SLAssetTypeToContentType((int)asset.Type); |
355 | sha = GetSHA256Hash(asset.Data); | 377 | sha = GetSHA256Hash(asset.Data); |
356 | m_log.InfoFormat("[FALLBACK]: Added asset {0} from fallback to local store", id); | 378 | m_log.InfoFormat("[FSASSETS]: Added asset {0} from fallback to local store", id); |
357 | Store(asset); | 379 | Store(asset); |
358 | } | 380 | } |
359 | } | 381 | } |
360 | if (asset == null) | 382 | if (asset == null) |
361 | m_missingAssetsFS++; | 383 | m_missingAssetsFS++; |
362 | // m_log.InfoFormat("[ASSET]: Asset {0}, hash {1} not found in FS", id, hash); | 384 | // m_log.InfoFormat("[FSASSETS]: Asset {0}, hash {1} not found in FS", id, hash); |
363 | else | 385 | else |
364 | return asset; | 386 | return asset; |
365 | } | 387 | } |
@@ -649,7 +671,7 @@ namespace OpenSim.Services.FSAssetService | |||
649 | { | 671 | { |
650 | count = Convert.ToInt32(args[4]); | 672 | count = Convert.ToInt32(args[4]); |
651 | } | 673 | } |
652 | m_DataConnector.Import(conn, table, start, count, force, new StoreDelegate(Store)); | 674 | m_DataConnector.Import(conn, table, start, count, force, new FSStoreDelegate(Store)); |
653 | } | 675 | } |
654 | } | 676 | } |
655 | 677 | ||