aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim/Data/Migration.cs
diff options
context:
space:
mode:
authorAlexRa2010-05-16 16:22:38 +0300
committerAlexRa2010-05-16 16:22:38 +0300
commit8b6a295874124f70146ae79f01281f2067068a82 (patch)
tree8088a830620832617a42311d871ab00c4fe9bbc7 /OpenSim/Data/Migration.cs
parentMerge branch 'master' of ssh://MyConnection/var/git/opensim (diff)
downloadopensim-SC_OLD-8b6a295874124f70146ae79f01281f2067068a82.zip
opensim-SC_OLD-8b6a295874124f70146ae79f01281f2067068a82.tar.gz
opensim-SC_OLD-8b6a295874124f70146ae79f01281f2067068a82.tar.bz2
opensim-SC_OLD-8b6a295874124f70146ae79f01281f2067068a82.tar.xz
Migration.cs supports single-file migration history format
Scans for migration resources in either old-style "scattered" (one file per version) or new-style "integrated" format (single file "Resources/{StoreName}.migrations[.nnn]") with ":VERSION nnn" sections). In the new-style migrations it also recognizes ':GO' separators for parts of the SQL script that must be sent to the server separately. The old-style migrations are loaded each in one piece and don't support the ':GO' feature. Status: TESTED and works fine in all modes!
Diffstat (limited to '')
-rw-r--r--OpenSim/Data/Migration.cs337
1 files changed, 233 insertions, 104 deletions
diff --git a/OpenSim/Data/Migration.cs b/OpenSim/Data/Migration.cs
index 06defe4..7980c35 100644
--- a/OpenSim/Data/Migration.cs
+++ b/OpenSim/Data/Migration.cs
@@ -70,61 +70,111 @@ namespace OpenSim.Data
70 70
71 public class Migration 71 public class Migration
72 { 72 {
73 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); 73 protected static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
74 74
75 private string _type; 75 protected string _type;
76 private DbConnection _conn; 76 protected DbConnection _conn;
77 // private string _subtype; 77 protected Assembly _assem;
78 private Assembly _assem;
79 private Regex _match;
80 78
81 private static readonly string _migrations_create = "create table migrations(name varchar(100), version int)"; 79 private Regex _match_old;
82 // private static readonly string _migrations_init = "insert into migrations values('migrations', 1)"; 80 private Regex _match_new;
83 // private static readonly string _migrations_find = "select version from migrations where name='migrations'";
84 81
82 /// <summary>Have the parameterless constructor just so we can specify it as a generic parameter with the new() constraint.
83 /// Currently this is only used in the tests. A Migration instance created this way must be then
84 /// initialized with Initialize(). Regular creation should be through the parameterized constructors.
85 /// </summary>
86 public Migration()
87 {
88 }
85 89
86 public Migration(DbConnection conn, Assembly assem, string type) 90 public Migration(DbConnection conn, Assembly assem, string subtype, string type)
87 { 91 {
88 _type = type; 92 Initialize(conn, assem, type, subtype);
89 _conn = conn;
90 _assem = assem;
91 _match = new Regex(@"\.(\d\d\d)_" + _type + @"\.sql");
92 Initialize();
93 } 93 }
94 94
95 public Migration(DbConnection conn, Assembly assem, string subtype, string type) 95 public Migration(DbConnection conn, Assembly assem, string type)
96 {
97 Initialize(conn, assem, type, "");
98 }
99
100 /// <summary>Must be called after creating with the parameterless constructor.
101 /// NOTE that the Migration class now doesn't access database in any way during initialization.
102 /// Specifically, it won't check if the [migrations] table exists. Such checks are done later:
103 /// automatically on Update(), or you can explicitly call InitMigrationsTable().
104 /// </summary>
105 /// <param name="conn"></param>
106 /// <param name="assem"></param>
107 /// <param name="subtype"></param>
108 /// <param name="type"></param>
109 public void Initialize (DbConnection conn, Assembly assem, string type, string subtype)
96 { 110 {
97 _type = type; 111 _type = type;
98 _conn = conn; 112 _conn = conn;
99 _assem = assem; 113 _assem = assem;
100 _match = new Regex(subtype + @"\.(\d\d\d)_" + _type + @"\.sql"); 114 _match_old = new Regex(subtype + @"\.(\d\d\d)_" + _type + @"\.sql");
101 Initialize(); 115 string s = String.IsNullOrEmpty(subtype) ? _type : _type + @"\." + subtype;
116 _match_new = new Regex(@"\." + s + @"\.migrations(?:\.(?<ver>\d+)$|.*)");
102 } 117 }
103 118
104 private void Initialize() 119 public void InitMigrationsTable()
105 { 120 {
106 // clever, eh, we figure out which migrations version we are 121 // NOTE: normally when the [migrations] table is created, the version record for 'migrations' is
107 int migration_version = FindVersion(_conn, "migrations"); 122 // added immediately. However, if for some reason the table is there but empty, we want to handle that as well.
108 123 int ver = FindVersion(_conn, "migrations");
109 if (migration_version > 0) 124 if (ver <= 0) // -1 = no table, 0 = no version record
110 return; 125 {
126 if( ver < 0 )
127 ExecuteScript("create table migrations(name varchar(100), version int)");
128 InsertVersion("migrations", 1);
129 }
130 }
111 131
112 // If not, create the migration tables 132 /// <summary>Executes a script, possibly in a database-specific way.
113 using (DbCommand cmd = _conn.CreateCommand()) 133 /// It can be redefined for a specific DBMS, if necessary. Specifically,
134 /// to avoid problems with proc definitions in MySQL, we must use
135 /// MySqlScript class instead of just DbCommand. We don't want to bring
136 /// MySQL references here, so instead define a MySQLMigration class
137 /// in OpenSim.Data.MySQL
138 /// </summary>
139 /// <param name="conn"></param>
140 /// <param name="script">Array of strings, one-per-batch (often just one)</param>
141 protected virtual void ExecuteScript(DbConnection conn, string[] script)
142 {
143 using (DbCommand cmd = conn.CreateCommand())
114 { 144 {
115 cmd.CommandText = _migrations_create; 145 cmd.CommandTimeout = 0;
116 cmd.ExecuteNonQuery(); 146 foreach (string sql in script)
147 {
148 cmd.CommandText = sql;
149 cmd.ExecuteNonQuery();
150 }
117 } 151 }
152 }
153
154 protected void ExecuteScript(DbConnection conn, string sql)
155 {
156 ExecuteScript(conn, new string[]{sql});
157 }
158
159 protected void ExecuteScript(string sql)
160 {
161 ExecuteScript(_conn, sql);
162 }
118 163
119 InsertVersion("migrations", 1); 164 protected void ExecuteScript(string[] script)
165 {
166 ExecuteScript(_conn, script);
120 } 167 }
121 168
169
170
122 public void Update() 171 public void Update()
123 { 172 {
124 int version = 0; 173 InitMigrationsTable();
125 version = FindVersion(_conn, _type);
126 174
127 SortedList<int, string> migrations = GetMigrationsAfter(version); 175 int version = FindVersion(_conn, _type);
176
177 SortedList<int, string[]> migrations = GetMigrationsAfter(version);
128 if (migrations.Count < 1) 178 if (migrations.Count < 1)
129 return; 179 return;
130 180
@@ -132,57 +182,41 @@ namespace OpenSim.Data
132 m_log.InfoFormat("[MIGRATIONS] Upgrading {0} to latest revision {1}.", _type, migrations.Keys[migrations.Count - 1]); 182 m_log.InfoFormat("[MIGRATIONS] Upgrading {0} to latest revision {1}.", _type, migrations.Keys[migrations.Count - 1]);
133 m_log.Info("[MIGRATIONS] NOTE: this may take a while, don't interupt this process!"); 183 m_log.Info("[MIGRATIONS] NOTE: this may take a while, don't interupt this process!");
134 184
135 using (DbCommand cmd = _conn.CreateCommand()) 185 foreach (KeyValuePair<int, string[]> kvp in migrations)
136 { 186 {
137 foreach (KeyValuePair<int, string> kvp in migrations) 187 int newversion = kvp.Key;
188 // we need to up the command timeout to infinite as we might be doing long migrations.
189
190 /* [AlexRa 01-May-10]: We can't always just run any SQL in a single batch (= ExecuteNonQuery()). Things like
191 * stored proc definitions might have to be sent to the server each in a separate batch.
192 * This is certainly so for MS SQL; not sure how the MySQL connector sorts out the mess
193 * with 'delimiter @@'/'delimiter ;' around procs. So each "script" this code executes now is not
194 * a single string, but an array of strings, executed separately.
195 */
196 try
138 { 197 {
139 int newversion = kvp.Key; 198 ExecuteScript(kvp.Value);
140 cmd.CommandText = kvp.Value; 199 }
141 // we need to up the command timeout to infinite as we might be doing long migrations. 200 catch (Exception e)
142 cmd.CommandTimeout = 0; 201 {
143 try 202 m_log.DebugFormat("[MIGRATIONS] Cmd was {0}", kvp.Value.ToString());
144 { 203 m_log.DebugFormat("[MIGRATIONS]: An error has occurred in the migration {0}.\n This may mean you could see errors trying to run OpenSim. If you see database related errors, you will need to fix the issue manually. Migration aborted.", e.Message);
145 cmd.ExecuteNonQuery(); 204 ExecuteScript("ROLLBACK;");
146 } 205 return;
147 catch (Exception e) 206 }
148 {
149 m_log.DebugFormat("[MIGRATIONS] Cmd was {0}", cmd.CommandText);
150 m_log.DebugFormat("[MIGRATIONS]: An error has occurred in the migration {0}.\n This may mean you could see errors trying to run OpenSim. If you see database related errors, you will need to fix the issue manually. Continuing.", e.Message);
151 cmd.CommandText = "ROLLBACK;";
152 cmd.ExecuteNonQuery();
153 }
154 207
155 if (version == 0) 208 if (version == 0)
156 { 209 {
157 InsertVersion(_type, newversion); 210 InsertVersion(_type, newversion);
158 }
159 else
160 {
161 UpdateVersion(_type, newversion);
162 }
163 version = newversion;
164 } 211 }
212 else
213 {
214 UpdateVersion(_type, newversion);
215 }
216 version = newversion;
165 } 217 }
166 } 218 }
167 219
168 // private int MaxVersion()
169 // {
170 // int max = 0;
171 // string[] names = _assem.GetManifestResourceNames();
172
173 // foreach (string s in names)
174 // {
175 // Match m = _match.Match(s);
176 // if (m.Success)
177 // {
178 // int MigrationVersion = int.Parse(m.Groups[1].ToString());
179 // if (MigrationVersion > max)
180 // max = MigrationVersion;
181 // }
182 // }
183 // return max;
184 // }
185
186 public int Version 220 public int Version
187 { 221 {
188 get { return FindVersion(_conn, _type); } 222 get { return FindVersion(_conn, _type); }
@@ -206,7 +240,7 @@ namespace OpenSim.Data
206 try 240 try
207 { 241 {
208 cmd.CommandText = "select version from migrations where name='" + type + "' order by version desc"; 242 cmd.CommandText = "select version from migrations where name='" + type + "' order by version desc";
209 using (IDataReader reader = cmd.ExecuteReader()) 243 using (DbDataReader reader = cmd.ExecuteReader())
210 { 244 {
211 if (reader.Read()) 245 if (reader.Read())
212 { 246 {
@@ -217,7 +251,8 @@ namespace OpenSim.Data
217 } 251 }
218 catch 252 catch
219 { 253 {
220 // Something went wrong, so we're version 0 254 // Something went wrong (probably no table), so we're at version -1
255 version = -1;
221 } 256 }
222 } 257 }
223 return version; 258 return version;
@@ -225,57 +260,151 @@ namespace OpenSim.Data
225 260
226 private void InsertVersion(string type, int version) 261 private void InsertVersion(string type, int version)
227 { 262 {
228 using (DbCommand cmd = _conn.CreateCommand()) 263 m_log.InfoFormat("[MIGRATIONS]: Creating {0} at version {1}", type, version);
229 { 264 ExecuteScript("insert into migrations(name, version) values('" + type + "', " + version + ")");
230 cmd.CommandText = "insert into migrations(name, version) values('" + type + "', " + version + ")";
231 m_log.InfoFormat("[MIGRATIONS]: Creating {0} at version {1}", type, version);
232 cmd.ExecuteNonQuery();
233 }
234 } 265 }
235 266
236 private void UpdateVersion(string type, int version) 267 private void UpdateVersion(string type, int version)
237 { 268 {
238 using (DbCommand cmd = _conn.CreateCommand()) 269 m_log.InfoFormat("[MIGRATIONS]: Updating {0} to version {1}", type, version);
239 { 270 ExecuteScript("update migrations set version=" + version + " where name='" + type + "'");
240 cmd.CommandText = "update migrations set version=" + version + " where name='" + type + "'";
241 m_log.InfoFormat("[MIGRATIONS]: Updating {0} to version {1}", type, version);
242 cmd.ExecuteNonQuery();
243 }
244 } 271 }
245 272
246 // private SortedList<int, string> GetAllMigrations() 273 private delegate void FlushProc();
247 // {
248 // return GetMigrationsAfter(0);
249 // }
250 274
251 private SortedList<int, string> GetMigrationsAfter(int after) 275 /// <summary>Scans for migration resources in either old-style "scattered" (one file per version)
276 /// or new-style "integrated" format (single file with ":VERSION nnn" sections).
277 /// In the new-style migrations it also recognizes ':GO' separators for parts of the SQL script
278 /// that must be sent to the server separately. The old-style migrations are loaded each in one piece
279 /// and don't support the ':GO' feature.
280 /// </summary>
281 /// <param name="after">The version we are currently at. Scan for any higher versions</param>
282 /// <returns>A list of string arrays, representing the scripts.</returns>
283 private SortedList<int, string[]> GetMigrationsAfter(int after)
252 { 284 {
285 SortedList<int, string[]> migrations = new SortedList<int, string[]>();
286
253 string[] names = _assem.GetManifestResourceNames(); 287 string[] names = _assem.GetManifestResourceNames();
254 SortedList<int, string> migrations = new SortedList<int, string>(); 288 if( names.Length == 0 ) // should never happen
255 // because life is funny if we don't 289 return migrations;
256 Array.Sort(names); 290
291 Array.Sort(names); // we want all the migrations ordered
292
293 int nLastVerFound = 0;
294 Match m = null;
295 string sFile = Array.FindLast(names, nm => { m = _match_new.Match(nm); return m.Success; }); // ; nm.StartsWith(sPrefix, StringComparison.InvariantCultureIgnoreCase
296
297 if( (m != null) && !String.IsNullOrEmpty(sFile) )
298 {
299 /* The filename should be '<StoreName>.migrations[.NNN]' where NNN
300 * is the last version number defined in the file. If the '.NNN' part is recognized, the code can skip
301 * the file without looking inside if we have a higher version already. Without the suffix we read
302 * the file anyway and use the version numbers inside. Any unrecognized suffix (such as '.sql')
303 * is valid but ignored.
304 *
305 * NOTE that we expect only one 'merged' migration file. If there are several, we take the last one.
306 * If you are numbering them, leave only the latest one in the project or at least make sure they numbered
307 * to come up in the correct order (e.g. 'SomeStore.migrations.001' rather than 'SomeStore.migrations.1')
308 */
309
310 if (m.Groups.Count > 1 && int.TryParse(m.Groups[1].Value, out nLastVerFound))
311 {
312 if( nLastVerFound <= after )
313 goto scan_old_style;
314 }
315
316 System.Text.StringBuilder sb = new System.Text.StringBuilder(4096);
317 int nVersion = -1;
318
319 List<string> script = new List<string>();
320
321 FlushProc flush = delegate()
322 {
323 if (sb.Length > 0) // last SQL stmt to script list
324 {
325 script.Add(sb.ToString());
326 sb.Length = 0;
327 }
328
329 if ( (nVersion > 0) && (nVersion > after) && (script.Count > 0) && !migrations.ContainsKey(nVersion)) // script to the versioned script list
330 {
331 migrations[nVersion] = script.ToArray();
332 }
333 script.Clear();
334 };
257 335
336 using (Stream resource = _assem.GetManifestResourceStream(sFile))
337 using (StreamReader resourceReader = new StreamReader(resource))
338 {
339 int nLineNo = 0;
340 while (!resourceReader.EndOfStream)
341 {
342 string sLine = resourceReader.ReadLine();
343 nLineNo++;
344
345 if( String.IsNullOrEmpty(sLine) || sLine.StartsWith("#") ) // ignore a comment or empty line
346 continue;
347
348 if (sLine.Trim().Equals(":GO", StringComparison.InvariantCultureIgnoreCase))
349 {
350 if (sb.Length == 0) continue;
351 if (nVersion > after)
352 script.Add(sb.ToString());
353 sb.Length = 0;
354 continue;
355 }
356
357 if (sLine.StartsWith(":VERSION ", StringComparison.InvariantCultureIgnoreCase)) // ":VERSION nnn"
358 {
359 flush();
360
361 int n = sLine.IndexOf('#'); // Comment is allowed in version sections, ignored
362 if (n >= 0)
363 sLine = sLine.Substring(0, n);
364
365 if (!int.TryParse(sLine.Substring(9).Trim(), out nVersion))
366 {
367 m_log.ErrorFormat("[MIGRATIONS]: invalid version marker at {0}: line {1}. Migration failed!", sFile, nLineNo);
368 break;
369 }
370 }
371 else
372 {
373 sb.AppendLine(sLine);
374 }
375 }
376 flush();
377
378 // If there are scattered migration files as well, only look for those with higher version numbers.
379 if (after < nVersion)
380 after = nVersion;
381 }
382 }
383
384scan_old_style:
385 // scan "old style" migration pieces anyway, ignore any versions already filled from the single file
258 foreach (string s in names) 386 foreach (string s in names)
259 { 387 {
260 Match m = _match.Match(s); 388 m = _match_old.Match(s);
261 if (m.Success) 389 if (m.Success)
262 { 390 {
263 int version = int.Parse(m.Groups[1].ToString()); 391 int version = int.Parse(m.Groups[1].ToString());
264 if (version > after) 392 if ( (version > after) && !migrations.ContainsKey(version) )
265 { 393 {
266 using (Stream resource = _assem.GetManifestResourceStream(s)) 394 using (Stream resource = _assem.GetManifestResourceStream(s))
267 { 395 {
268 using (StreamReader resourceReader = new StreamReader(resource)) 396 using (StreamReader resourceReader = new StreamReader(resource))
269 { 397 {
270 string resourceString = resourceReader.ReadToEnd(); 398 string sql = resourceReader.ReadToEnd();
271 migrations.Add(version, resourceString); 399 migrations.Add(version, new string[]{sql});
272 } 400 }
273 } 401 }
274 } 402 }
275 } 403 }
276 } 404 }
277 405
278 if (migrations.Count < 1) { 406 if (migrations.Count < 1)
407 {
279 m_log.InfoFormat("[MIGRATIONS]: {0} up to date, no migrations to apply", _type); 408 m_log.InfoFormat("[MIGRATIONS]: {0} up to date, no migrations to apply", _type);
280 } 409 }
281 return migrations; 410 return migrations;