aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorDavid Walter Seikel2016-11-28 00:08:50 +1000
committerDavid Walter Seikel2016-11-28 00:08:50 +1000
commit2755b5134e22cf8f04afcecf44d9d3b6ded2e566 (patch)
treecb0817526d6025c10a8c090862f386a9630439eb
parentScripts run much faster now, there might be more events flying around. (diff)
downloadopensim-SC_OLD-2755b5134e22cf8f04afcecf44d9d3b6ded2e566.zip
opensim-SC_OLD-2755b5134e22cf8f04afcecf44d9d3b6ded2e566.tar.gz
opensim-SC_OLD-2755b5134e22cf8f04afcecf44d9d3b6ded2e566.tar.bz2
opensim-SC_OLD-2755b5134e22cf8f04afcecf44d9d3b6ded2e566.tar.xz
Add asset fixing Perl script, though likely to convert it to Lua later.
Diffstat (limited to '')
-rwxr-xr-xscripts/fix_some_assets.pl227
1 files changed, 227 insertions, 0 deletions
diff --git a/scripts/fix_some_assets.pl b/scripts/fix_some_assets.pl
new file mode 100755
index 0000000..08fdcb2
--- /dev/null
+++ b/scripts/fix_some_assets.pl
@@ -0,0 +1,227 @@
1#!/usr/bin/perl
2use strict;
3use warnings;
4
5
6# Took almost two and a half hours on IG.
7
8
9=pod
10# 7/30/2015 3:58PM
11Rev B - cleanup
12Rev C - added in missing UPDATE
13
14 quthor Jeff Kelley <opensim@pescadoo.net>
15 mods by Fred Beckhusen <fred@mitsi.com> aka Ferd Frederix
16
17 This version has minimum Perl module dependencies.
18
19 This script connects to an Opensim asset database and scans and fixed bug Mantis-7514
20 The files that are fixed have multiple xmlns : in it, such as xmlns:xmlns:
21
22 http://opensimulator.org/mantis/view.php?id=7514
23
24############################################################################
25# DO A BACKUP BEFORE YOU ATTEMPT THIS. DO A DRY RUN WITH UPDATE SET TO ZERO #
26#############################################################################
27
28 Set variable UPDATE to a 1 to write updates to the db. A 0 will show you what it would change and also save the bad
29 data to disk in a folder name 'corrupt' for you to examine.
30
31 For safety, we will never update a record unless the UPDATE flag is on.
32
33#############################################################
34# DO NOT RUN THIS UNTIL YOU SET THE CONSTANTS BELOW #
35#############################################################
36
37=cut
38
39# these are typically found in StandaloneCommon.ini or GridCommon.ini
40
41my $username = '*****';
42my $password = '*****';
43my $hostname = '*****'; # probably you use localhost for standalone, but it could be an IP address or a DNS name for a asset server.
44my $robustDB = '*****'; # your MySQl DB name
45
46# for safety, we will never update a record unless the UPDATE flag is on.
47use constant LISTALL => 0; # show all assets as they are scanned
48use constant UPDATE => 0; # set to 1 if you want to actually update the database
49use constant MAX => 16000000; # max size of blob from your MySQL ini - see My.ini : max_allowed_packet = 16M, You may need to set this to 64 MB if you have large assets. My largest was over 10 MB..jeepers!
50use constant GETMAX => 1; # set to 1 to query MAX object size (see above line) automatically - much slower to start, but less memory is used overall both at the server and at the client
51use constant MANTIS7514 => '/tmp/fix_some_assets.ini'; # path to a temp file to keep track of each run. Delete the file to start at the beginning
52
53# all these modules should already be in core. If not, run "cpan name_of_module', or just go get Strawberry Perl.
54use v5.10; # so we can say, and not need to add a newline
55use DBI; # database I/O
56$|=1; #no buffering STDIO
57
58my $counter = 0; # count of xmlmns corruption
59my $havedone = 0;
60my $OddCounter = 0; # other corrupt data
61
62mkdir 'corrupt'; # save the data in this folder
63
64my $dbh = DBI->connect(
65 "dbi:mysql:dbname=$robustDB;host=$hostname",
66 $username, $password,{RaiseError => 1 , LongReadLen => MAX, LongTruncOk => 0,AutoCommit => 1},
67 ) or die $DBI::errstr;
68
69# Get the largest blob in table data in the DB and set our size of memory buffers to that.
70if (GETMAX) {
71 my $max_len = GetLenData ();
72 say "Largest object is $max_len bytes";
73 $dbh->{LongReadLen} = $max_len;
74}
75
76my $todo;
77my $objectsList = GetAssetList(); # get a hash of all objects
78$todo = scalar @$objectsList if ref $objectsList; # count them
79say "Found $todo objects";
80
81for (@$objectsList) {
82 my $obj_uuid = $_->{id};
83 my $obj_name = $_->{name};
84 my $obj_time = $_->{create_time};
85
86 $havedone++; # keep track of how many done
87
88 my $obj_data = GetAssetData ($obj_uuid); # grab the blob
89
90 ValidateXML ($obj_data,$obj_uuid, $obj_name); # repair it and check it
91
92 #save where we are at so we can be stopped and run again some other rainy day
93 open(my $in_file, ">",MANTIS7514) || die 'Cannot save the last file date';
94 print $in_file $obj_time;
95 close $in_file;
96}
97
98say "Found $counter corrupt xmlns objects";
99say "Found $OddCounter other possibly corrupted objects";
100say "Done!";
101
102
103sub GetAssetList {
104
105 my $answref;
106
107 my $time = 0;
108 # read the last run date from a file
109 if (open(my $in_file, "<",MANTIS7514))
110 {
111 $time = <$in_file> || 0;
112 close $in_file;
113 }
114 my $query = "SELECT id,name, create_time FROM $robustDB.assets WHERE assetType=6 and create_time > $time order by create_time asc;";
115 $answref = $dbh->selectall_arrayref ($query, { Slice => {} });
116
117 return $answref;
118}
119
120sub GetAssetData {
121 my $uuid = shift;
122 my $query = "SELECT data FROM $robustDB.assets WHERE id='$uuid';";
123 my $answr = $dbh->selectall_arrayref ($query, { Slice => {} });
124 return @$answr[0]->{data};
125}
126
127sub UpdateData {
128 return unless UPDATE;
129 my $uuid = shift;
130 my $data = shift;
131
132 my $sth = $dbh->prepare("UPDATE $robustDB.assets set data = ? WHERE id = ?;");
133 $sth->execute($data, $uuid) or die $DBI::errstr;;
134}
135sub GetLenData {
136 my $len = $dbh->selectrow_array("SELECT MAX(OCTET_LENGTH(data)) FROM $robustDB.assets WHERE assetType=6;");
137}
138
139
140sub ValidateXML {
141 my $data = shift;
142 my $obj_uuid = shift;
143 my $obj_name = shift || '';
144
145 #### FORCES AN ERROR FOR DEBUG $data =~ s/xmlns:/xmlns:xmlns:/g; # the fix is in !!!!!!!!!
146
147 # Test for repeated xmlns, clever RegEx by Jeff Kelley.
148 my $corrupt = ($data =~ m/((xmlns:){2,}+)/g);
149 my $err = $1 || '';
150
151 # show them where we are at, if enabled or corrupted.
152 printf "%d/%d | %s | %s | %s\n", $havedone, $todo, $corrupt ? 'Bad ' : 'Ok ', $obj_uuid, $obj_name if ($corrupt || LISTALL);
153
154 if ($corrupt)
155 {
156 $counter++;
157
158 print ("Found $err\n", 'red') ;
159 my $original = $data; # so we can save it to disk later, if need be.
160
161 $data =~ s/(xmlns:){2,}+/xmlns:/g; # the fix is in
162
163 UpdateData($obj_uuid,$data) if UPDATE; # we update if we are enabled and had an error Rev C
164
165 save({
166 name => "$obj_uuid-$obj_name-before.txt",
167 uuid => $obj_uuid,
168 corrupt => $err,
169 data => $original,
170 type => 0,
171 });
172
173 save({
174 name => "$obj_uuid-$obj_name-after.txt",
175 uuid => $obj_uuid,
176 corrupt => $err,
177 data => $data,
178 type => 0,
179 });
180 }
181
182 return $corrupt; # in case the caller wants to know
183}
184
185sub save {
186 my $c = shift;
187
188 $c->{name} =~ s/[^A-Za-z0-9\-\.]//g; # make a safe file name
189
190 # may have to force an UTF-16LE or UTF-BE encoding here but my tests show the data is always UTF-8 or ASCII.
191
192 if (open(my $out_file, "> :encoding(UTF-8)",'corrupt/' . $c->{name})) {
193 binmode ($out_file,":encoding(UTF-8)");
194
195 $c->{data} =~ s/encoding="utf-8"/encoding="utf-16"/; # not right, but that's they way they coded the XML in the DB.
196
197 print $out_file $c->{data};
198 print $out_file "\0"; # zero termination is in the OAR, lets put it back
199 } else {
200 say "Failed to open file";
201 die;
202 }
203
204}
205
206__END__
207
208BSD License:
209Copyright (c) 2015
210Redistribution and use in source and binary forms, with or without
211modification, are permitted provided that the following conditions are met:
212* Redistributions of source code must retain the above copyright notice, this
213 list of conditions and the following disclaimer.
214* Redistributions in binary form must reproduce the above copyright notice,
215 this list of conditions and the following disclaimer in the documentation
216 and/or other materials provided with the distribution.
217THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
218AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
219IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
220DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
221FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
222DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
223SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
224CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
225OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
226OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
227Space here is intentionally left blank for note taking