diff options
author | AlexRa | 2010-05-16 16:22:38 +0300 |
---|---|---|
committer | AlexRa | 2010-05-18 22:15:56 +0300 |
commit | f4450ccf4f76db7f6e18c483ac48f30e0907eb2a (patch) | |
tree | 06822de064bf13c83f32f5e076886850b4a9df54 /OpenSim | |
parent | Allow remote admin to be used on a different port from the main region port (diff) | |
download | opensim-SC_OLD-f4450ccf4f76db7f6e18c483ac48f30e0907eb2a.zip opensim-SC_OLD-f4450ccf4f76db7f6e18c483ac48f30e0907eb2a.tar.gz opensim-SC_OLD-f4450ccf4f76db7f6e18c483ac48f30e0907eb2a.tar.bz2 opensim-SC_OLD-f4450ccf4f76db7f6e18c483ac48f30e0907eb2a.tar.xz |
Migration.cs supports single-file migration history format
Scans for migration resources in either old-style "scattered" (one file per version)
or new-style "integrated" format (single file "Resources/{StoreName}.migrations[.nnn]") with ":VERSION nnn" sections).
In the new-style migrations it also recognizes ':GO' separators for parts of the SQL script
that must be sent to the server separately. The old-style migrations are loaded each in one piece
and don't support the ':GO' feature.
Status: TESTED and works fine in all modes!
Diffstat (limited to '')
-rw-r--r-- | OpenSim/Data/Migration.cs | 337 |
1 files changed, 233 insertions, 104 deletions
diff --git a/OpenSim/Data/Migration.cs b/OpenSim/Data/Migration.cs index 06defe4..7980c35 100644 --- a/OpenSim/Data/Migration.cs +++ b/OpenSim/Data/Migration.cs | |||
@@ -70,61 +70,111 @@ namespace OpenSim.Data | |||
70 | 70 | ||
71 | public class Migration | 71 | public class Migration |
72 | { | 72 | { |
73 | private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); | 73 | protected static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
74 | 74 | ||
75 | private string _type; | 75 | protected string _type; |
76 | private DbConnection _conn; | 76 | protected DbConnection _conn; |
77 | // private string _subtype; | 77 | protected Assembly _assem; |
78 | private Assembly _assem; | ||
79 | private Regex _match; | ||
80 | 78 | ||
81 | private static readonly string _migrations_create = "create table migrations(name varchar(100), version int)"; | 79 | private Regex _match_old; |
82 | // private static readonly string _migrations_init = "insert into migrations values('migrations', 1)"; | 80 | private Regex _match_new; |
83 | // private static readonly string _migrations_find = "select version from migrations where name='migrations'"; | ||
84 | 81 | ||
82 | /// <summary>Have the parameterless constructor just so we can specify it as a generic parameter with the new() constraint. | ||
83 | /// Currently this is only used in the tests. A Migration instance created this way must be then | ||
84 | /// initialized with Initialize(). Regular creation should be through the parameterized constructors. | ||
85 | /// </summary> | ||
86 | public Migration() | ||
87 | { | ||
88 | } | ||
85 | 89 | ||
86 | public Migration(DbConnection conn, Assembly assem, string type) | 90 | public Migration(DbConnection conn, Assembly assem, string subtype, string type) |
87 | { | 91 | { |
88 | _type = type; | 92 | Initialize(conn, assem, type, subtype); |
89 | _conn = conn; | ||
90 | _assem = assem; | ||
91 | _match = new Regex(@"\.(\d\d\d)_" + _type + @"\.sql"); | ||
92 | Initialize(); | ||
93 | } | 93 | } |
94 | 94 | ||
95 | public Migration(DbConnection conn, Assembly assem, string subtype, string type) | 95 | public Migration(DbConnection conn, Assembly assem, string type) |
96 | { | ||
97 | Initialize(conn, assem, type, ""); | ||
98 | } | ||
99 | |||
100 | /// <summary>Must be called after creating with the parameterless constructor. | ||
101 | /// NOTE that the Migration class now doesn't access database in any way during initialization. | ||
102 | /// Specifically, it won't check if the [migrations] table exists. Such checks are done later: | ||
103 | /// automatically on Update(), or you can explicitly call InitMigrationsTable(). | ||
104 | /// </summary> | ||
105 | /// <param name="conn"></param> | ||
106 | /// <param name="assem"></param> | ||
107 | /// <param name="subtype"></param> | ||
108 | /// <param name="type"></param> | ||
109 | public void Initialize (DbConnection conn, Assembly assem, string type, string subtype) | ||
96 | { | 110 | { |
97 | _type = type; | 111 | _type = type; |
98 | _conn = conn; | 112 | _conn = conn; |
99 | _assem = assem; | 113 | _assem = assem; |
100 | _match = new Regex(subtype + @"\.(\d\d\d)_" + _type + @"\.sql"); | 114 | _match_old = new Regex(subtype + @"\.(\d\d\d)_" + _type + @"\.sql"); |
101 | Initialize(); | 115 | string s = String.IsNullOrEmpty(subtype) ? _type : _type + @"\." + subtype; |
116 | _match_new = new Regex(@"\." + s + @"\.migrations(?:\.(?<ver>\d+)$|.*)"); | ||
102 | } | 117 | } |
103 | 118 | ||
104 | private void Initialize() | 119 | public void InitMigrationsTable() |
105 | { | 120 | { |
106 | // clever, eh, we figure out which migrations version we are | 121 | // NOTE: normally when the [migrations] table is created, the version record for 'migrations' is |
107 | int migration_version = FindVersion(_conn, "migrations"); | 122 | // added immediately. However, if for some reason the table is there but empty, we want to handle that as well. |
108 | 123 | int ver = FindVersion(_conn, "migrations"); | |
109 | if (migration_version > 0) | 124 | if (ver <= 0) // -1 = no table, 0 = no version record |
110 | return; | 125 | { |
126 | if( ver < 0 ) | ||
127 | ExecuteScript("create table migrations(name varchar(100), version int)"); | ||
128 | InsertVersion("migrations", 1); | ||
129 | } | ||
130 | } | ||
111 | 131 | ||
112 | // If not, create the migration tables | 132 | /// <summary>Executes a script, possibly in a database-specific way. |
113 | using (DbCommand cmd = _conn.CreateCommand()) | 133 | /// It can be redefined for a specific DBMS, if necessary. Specifically, |
134 | /// to avoid problems with proc definitions in MySQL, we must use | ||
135 | /// MySqlScript class instead of just DbCommand. We don't want to bring | ||
136 | /// MySQL references here, so instead define a MySQLMigration class | ||
137 | /// in OpenSim.Data.MySQL | ||
138 | /// </summary> | ||
139 | /// <param name="conn"></param> | ||
140 | /// <param name="script">Array of strings, one-per-batch (often just one)</param> | ||
141 | protected virtual void ExecuteScript(DbConnection conn, string[] script) | ||
142 | { | ||
143 | using (DbCommand cmd = conn.CreateCommand()) | ||
114 | { | 144 | { |
115 | cmd.CommandText = _migrations_create; | 145 | cmd.CommandTimeout = 0; |
116 | cmd.ExecuteNonQuery(); | 146 | foreach (string sql in script) |
147 | { | ||
148 | cmd.CommandText = sql; | ||
149 | cmd.ExecuteNonQuery(); | ||
150 | } | ||
117 | } | 151 | } |
152 | } | ||
153 | |||
154 | protected void ExecuteScript(DbConnection conn, string sql) | ||
155 | { | ||
156 | ExecuteScript(conn, new string[]{sql}); | ||
157 | } | ||
158 | |||
159 | protected void ExecuteScript(string sql) | ||
160 | { | ||
161 | ExecuteScript(_conn, sql); | ||
162 | } | ||
118 | 163 | ||
119 | InsertVersion("migrations", 1); | 164 | protected void ExecuteScript(string[] script) |
165 | { | ||
166 | ExecuteScript(_conn, script); | ||
120 | } | 167 | } |
121 | 168 | ||
169 | |||
170 | |||
122 | public void Update() | 171 | public void Update() |
123 | { | 172 | { |
124 | int version = 0; | 173 | InitMigrationsTable(); |
125 | version = FindVersion(_conn, _type); | ||
126 | 174 | ||
127 | SortedList<int, string> migrations = GetMigrationsAfter(version); | 175 | int version = FindVersion(_conn, _type); |
176 | |||
177 | SortedList<int, string[]> migrations = GetMigrationsAfter(version); | ||
128 | if (migrations.Count < 1) | 178 | if (migrations.Count < 1) |
129 | return; | 179 | return; |
130 | 180 | ||
@@ -132,57 +182,41 @@ namespace OpenSim.Data | |||
132 | m_log.InfoFormat("[MIGRATIONS] Upgrading {0} to latest revision {1}.", _type, migrations.Keys[migrations.Count - 1]); | 182 | m_log.InfoFormat("[MIGRATIONS] Upgrading {0} to latest revision {1}.", _type, migrations.Keys[migrations.Count - 1]); |
133 | m_log.Info("[MIGRATIONS] NOTE: this may take a while, don't interupt this process!"); | 183 | m_log.Info("[MIGRATIONS] NOTE: this may take a while, don't interupt this process!"); |
134 | 184 | ||
135 | using (DbCommand cmd = _conn.CreateCommand()) | 185 | foreach (KeyValuePair<int, string[]> kvp in migrations) |
136 | { | 186 | { |
137 | foreach (KeyValuePair<int, string> kvp in migrations) | 187 | int newversion = kvp.Key; |
188 | // we need to up the command timeout to infinite as we might be doing long migrations. | ||
189 | |||
190 | /* [AlexRa 01-May-10]: We can't always just run any SQL in a single batch (= ExecuteNonQuery()). Things like | ||
191 | * stored proc definitions might have to be sent to the server each in a separate batch. | ||
192 | * This is certainly so for MS SQL; not sure how the MySQL connector sorts out the mess | ||
193 | * with 'delimiter @@'/'delimiter ;' around procs. So each "script" this code executes now is not | ||
194 | * a single string, but an array of strings, executed separately. | ||
195 | */ | ||
196 | try | ||
138 | { | 197 | { |
139 | int newversion = kvp.Key; | 198 | ExecuteScript(kvp.Value); |
140 | cmd.CommandText = kvp.Value; | 199 | } |
141 | // we need to up the command timeout to infinite as we might be doing long migrations. | 200 | catch (Exception e) |
142 | cmd.CommandTimeout = 0; | 201 | { |
143 | try | 202 | m_log.DebugFormat("[MIGRATIONS] Cmd was {0}", kvp.Value.ToString()); |
144 | { | 203 | m_log.DebugFormat("[MIGRATIONS]: An error has occurred in the migration {0}.\n This may mean you could see errors trying to run OpenSim. If you see database related errors, you will need to fix the issue manually. Migration aborted.", e.Message); |
145 | cmd.ExecuteNonQuery(); | 204 | ExecuteScript("ROLLBACK;"); |
146 | } | 205 | return; |
147 | catch (Exception e) | 206 | } |
148 | { | ||
149 | m_log.DebugFormat("[MIGRATIONS] Cmd was {0}", cmd.CommandText); | ||
150 | m_log.DebugFormat("[MIGRATIONS]: An error has occurred in the migration {0}.\n This may mean you could see errors trying to run OpenSim. If you see database related errors, you will need to fix the issue manually. Continuing.", e.Message); | ||
151 | cmd.CommandText = "ROLLBACK;"; | ||
152 | cmd.ExecuteNonQuery(); | ||
153 | } | ||
154 | 207 | ||
155 | if (version == 0) | 208 | if (version == 0) |
156 | { | 209 | { |
157 | InsertVersion(_type, newversion); | 210 | InsertVersion(_type, newversion); |
158 | } | ||
159 | else | ||
160 | { | ||
161 | UpdateVersion(_type, newversion); | ||
162 | } | ||
163 | version = newversion; | ||
164 | } | 211 | } |
212 | else | ||
213 | { | ||
214 | UpdateVersion(_type, newversion); | ||
215 | } | ||
216 | version = newversion; | ||
165 | } | 217 | } |
166 | } | 218 | } |
167 | 219 | ||
168 | // private int MaxVersion() | ||
169 | // { | ||
170 | // int max = 0; | ||
171 | // string[] names = _assem.GetManifestResourceNames(); | ||
172 | |||
173 | // foreach (string s in names) | ||
174 | // { | ||
175 | // Match m = _match.Match(s); | ||
176 | // if (m.Success) | ||
177 | // { | ||
178 | // int MigrationVersion = int.Parse(m.Groups[1].ToString()); | ||
179 | // if (MigrationVersion > max) | ||
180 | // max = MigrationVersion; | ||
181 | // } | ||
182 | // } | ||
183 | // return max; | ||
184 | // } | ||
185 | |||
186 | public int Version | 220 | public int Version |
187 | { | 221 | { |
188 | get { return FindVersion(_conn, _type); } | 222 | get { return FindVersion(_conn, _type); } |
@@ -206,7 +240,7 @@ namespace OpenSim.Data | |||
206 | try | 240 | try |
207 | { | 241 | { |
208 | cmd.CommandText = "select version from migrations where name='" + type + "' order by version desc"; | 242 | cmd.CommandText = "select version from migrations where name='" + type + "' order by version desc"; |
209 | using (IDataReader reader = cmd.ExecuteReader()) | 243 | using (DbDataReader reader = cmd.ExecuteReader()) |
210 | { | 244 | { |
211 | if (reader.Read()) | 245 | if (reader.Read()) |
212 | { | 246 | { |
@@ -217,7 +251,8 @@ namespace OpenSim.Data | |||
217 | } | 251 | } |
218 | catch | 252 | catch |
219 | { | 253 | { |
220 | // Something went wrong, so we're version 0 | 254 | // Something went wrong (probably no table), so we're at version -1 |
255 | version = -1; | ||
221 | } | 256 | } |
222 | } | 257 | } |
223 | return version; | 258 | return version; |
@@ -225,57 +260,151 @@ namespace OpenSim.Data | |||
225 | 260 | ||
226 | private void InsertVersion(string type, int version) | 261 | private void InsertVersion(string type, int version) |
227 | { | 262 | { |
228 | using (DbCommand cmd = _conn.CreateCommand()) | 263 | m_log.InfoFormat("[MIGRATIONS]: Creating {0} at version {1}", type, version); |
229 | { | 264 | ExecuteScript("insert into migrations(name, version) values('" + type + "', " + version + ")"); |
230 | cmd.CommandText = "insert into migrations(name, version) values('" + type + "', " + version + ")"; | ||
231 | m_log.InfoFormat("[MIGRATIONS]: Creating {0} at version {1}", type, version); | ||
232 | cmd.ExecuteNonQuery(); | ||
233 | } | ||
234 | } | 265 | } |
235 | 266 | ||
236 | private void UpdateVersion(string type, int version) | 267 | private void UpdateVersion(string type, int version) |
237 | { | 268 | { |
238 | using (DbCommand cmd = _conn.CreateCommand()) | 269 | m_log.InfoFormat("[MIGRATIONS]: Updating {0} to version {1}", type, version); |
239 | { | 270 | ExecuteScript("update migrations set version=" + version + " where name='" + type + "'"); |
240 | cmd.CommandText = "update migrations set version=" + version + " where name='" + type + "'"; | ||
241 | m_log.InfoFormat("[MIGRATIONS]: Updating {0} to version {1}", type, version); | ||
242 | cmd.ExecuteNonQuery(); | ||
243 | } | ||
244 | } | 271 | } |
245 | 272 | ||
246 | // private SortedList<int, string> GetAllMigrations() | 273 | private delegate void FlushProc(); |
247 | // { | ||
248 | // return GetMigrationsAfter(0); | ||
249 | // } | ||
250 | 274 | ||
251 | private SortedList<int, string> GetMigrationsAfter(int after) | 275 | /// <summary>Scans for migration resources in either old-style "scattered" (one file per version) |
276 | /// or new-style "integrated" format (single file with ":VERSION nnn" sections). | ||
277 | /// In the new-style migrations it also recognizes ':GO' separators for parts of the SQL script | ||
278 | /// that must be sent to the server separately. The old-style migrations are loaded each in one piece | ||
279 | /// and don't support the ':GO' feature. | ||
280 | /// </summary> | ||
281 | /// <param name="after">The version we are currently at. Scan for any higher versions</param> | ||
282 | /// <returns>A list of string arrays, representing the scripts.</returns> | ||
283 | private SortedList<int, string[]> GetMigrationsAfter(int after) | ||
252 | { | 284 | { |
285 | SortedList<int, string[]> migrations = new SortedList<int, string[]>(); | ||
286 | |||
253 | string[] names = _assem.GetManifestResourceNames(); | 287 | string[] names = _assem.GetManifestResourceNames(); |
254 | SortedList<int, string> migrations = new SortedList<int, string>(); | 288 | if( names.Length == 0 ) // should never happen |
255 | // because life is funny if we don't | 289 | return migrations; |
256 | Array.Sort(names); | 290 | |
291 | Array.Sort(names); // we want all the migrations ordered | ||
292 | |||
293 | int nLastVerFound = 0; | ||
294 | Match m = null; | ||
295 | string sFile = Array.FindLast(names, nm => { m = _match_new.Match(nm); return m.Success; }); // ; nm.StartsWith(sPrefix, StringComparison.InvariantCultureIgnoreCase | ||
296 | |||
297 | if( (m != null) && !String.IsNullOrEmpty(sFile) ) | ||
298 | { | ||
299 | /* The filename should be '<StoreName>.migrations[.NNN]' where NNN | ||
300 | * is the last version number defined in the file. If the '.NNN' part is recognized, the code can skip | ||
301 | * the file without looking inside if we have a higher version already. Without the suffix we read | ||
302 | * the file anyway and use the version numbers inside. Any unrecognized suffix (such as '.sql') | ||
303 | * is valid but ignored. | ||
304 | * | ||
305 | * NOTE that we expect only one 'merged' migration file. If there are several, we take the last one. | ||
306 | * If you are numbering them, leave only the latest one in the project or at least make sure they numbered | ||
307 | * to come up in the correct order (e.g. 'SomeStore.migrations.001' rather than 'SomeStore.migrations.1') | ||
308 | */ | ||
309 | |||
310 | if (m.Groups.Count > 1 && int.TryParse(m.Groups[1].Value, out nLastVerFound)) | ||
311 | { | ||
312 | if( nLastVerFound <= after ) | ||
313 | goto scan_old_style; | ||
314 | } | ||
315 | |||
316 | System.Text.StringBuilder sb = new System.Text.StringBuilder(4096); | ||
317 | int nVersion = -1; | ||
318 | |||
319 | List<string> script = new List<string>(); | ||
320 | |||
321 | FlushProc flush = delegate() | ||
322 | { | ||
323 | if (sb.Length > 0) // last SQL stmt to script list | ||
324 | { | ||
325 | script.Add(sb.ToString()); | ||
326 | sb.Length = 0; | ||
327 | } | ||
328 | |||
329 | if ( (nVersion > 0) && (nVersion > after) && (script.Count > 0) && !migrations.ContainsKey(nVersion)) // script to the versioned script list | ||
330 | { | ||
331 | migrations[nVersion] = script.ToArray(); | ||
332 | } | ||
333 | script.Clear(); | ||
334 | }; | ||
257 | 335 | ||
336 | using (Stream resource = _assem.GetManifestResourceStream(sFile)) | ||
337 | using (StreamReader resourceReader = new StreamReader(resource)) | ||
338 | { | ||
339 | int nLineNo = 0; | ||
340 | while (!resourceReader.EndOfStream) | ||
341 | { | ||
342 | string sLine = resourceReader.ReadLine(); | ||
343 | nLineNo++; | ||
344 | |||
345 | if( String.IsNullOrEmpty(sLine) || sLine.StartsWith("#") ) // ignore a comment or empty line | ||
346 | continue; | ||
347 | |||
348 | if (sLine.Trim().Equals(":GO", StringComparison.InvariantCultureIgnoreCase)) | ||
349 | { | ||
350 | if (sb.Length == 0) continue; | ||
351 | if (nVersion > after) | ||
352 | script.Add(sb.ToString()); | ||
353 | sb.Length = 0; | ||
354 | continue; | ||
355 | } | ||
356 | |||
357 | if (sLine.StartsWith(":VERSION ", StringComparison.InvariantCultureIgnoreCase)) // ":VERSION nnn" | ||
358 | { | ||
359 | flush(); | ||
360 | |||
361 | int n = sLine.IndexOf('#'); // Comment is allowed in version sections, ignored | ||
362 | if (n >= 0) | ||
363 | sLine = sLine.Substring(0, n); | ||
364 | |||
365 | if (!int.TryParse(sLine.Substring(9).Trim(), out nVersion)) | ||
366 | { | ||
367 | m_log.ErrorFormat("[MIGRATIONS]: invalid version marker at {0}: line {1}. Migration failed!", sFile, nLineNo); | ||
368 | break; | ||
369 | } | ||
370 | } | ||
371 | else | ||
372 | { | ||
373 | sb.AppendLine(sLine); | ||
374 | } | ||
375 | } | ||
376 | flush(); | ||
377 | |||
378 | // If there are scattered migration files as well, only look for those with higher version numbers. | ||
379 | if (after < nVersion) | ||
380 | after = nVersion; | ||
381 | } | ||
382 | } | ||
383 | |||
384 | scan_old_style: | ||
385 | // scan "old style" migration pieces anyway, ignore any versions already filled from the single file | ||
258 | foreach (string s in names) | 386 | foreach (string s in names) |
259 | { | 387 | { |
260 | Match m = _match.Match(s); | 388 | m = _match_old.Match(s); |
261 | if (m.Success) | 389 | if (m.Success) |
262 | { | 390 | { |
263 | int version = int.Parse(m.Groups[1].ToString()); | 391 | int version = int.Parse(m.Groups[1].ToString()); |
264 | if (version > after) | 392 | if ( (version > after) && !migrations.ContainsKey(version) ) |
265 | { | 393 | { |
266 | using (Stream resource = _assem.GetManifestResourceStream(s)) | 394 | using (Stream resource = _assem.GetManifestResourceStream(s)) |
267 | { | 395 | { |
268 | using (StreamReader resourceReader = new StreamReader(resource)) | 396 | using (StreamReader resourceReader = new StreamReader(resource)) |
269 | { | 397 | { |
270 | string resourceString = resourceReader.ReadToEnd(); | 398 | string sql = resourceReader.ReadToEnd(); |
271 | migrations.Add(version, resourceString); | 399 | migrations.Add(version, new string[]{sql}); |
272 | } | 400 | } |
273 | } | 401 | } |
274 | } | 402 | } |
275 | } | 403 | } |
276 | } | 404 | } |
277 | 405 | ||
278 | if (migrations.Count < 1) { | 406 | if (migrations.Count < 1) |
407 | { | ||
279 | m_log.InfoFormat("[MIGRATIONS]: {0} up to date, no migrations to apply", _type); | 408 | m_log.InfoFormat("[MIGRATIONS]: {0} up to date, no migrations to apply", _type); |
280 | } | 409 | } |
281 | return migrations; | 410 | return migrations; |