1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
#!/usr/bin/perl
use strict;
use warnings;
# Took almost two and a half hours on IG.
=pod
# 7/30/2015 3:58PM
Rev B - cleanup
Rev C - added in missing UPDATE
quthor Jeff Kelley <opensim@pescadoo.net>
mods by Fred Beckhusen <fred@mitsi.com> aka Ferd Frederix
This version has minimum Perl module dependencies.
This script connects to an Opensim asset database and scans and fixed bug Mantis-7514
The files that are fixed have multiple xmlns : in it, such as xmlns:xmlns:
http://opensimulator.org/mantis/view.php?id=7514
############################################################################
# DO A BACKUP BEFORE YOU ATTEMPT THIS. DO A DRY RUN WITH UPDATE SET TO ZERO #
#############################################################################
Set variable UPDATE to a 1 to write updates to the db. A 0 will show you what it would change and also save the bad
data to disk in a folder name 'corrupt' for you to examine.
For safety, we will never update a record unless the UPDATE flag is on.
#############################################################
# DO NOT RUN THIS UNTIL YOU SET THE CONSTANTS BELOW #
#############################################################
=cut
# these are typically found in StandaloneCommon.ini or GridCommon.ini
my $username = '*****';
my $password = '*****';
my $hostname = '*****'; # probably you use localhost for standalone, but it could be an IP address or a DNS name for a asset server.
my $robustDB = '*****'; # your MySQl DB name
# for safety, we will never update a record unless the UPDATE flag is on.
use constant LISTALL => 0; # show all assets as they are scanned
use constant UPDATE => 0; # set to 1 if you want to actually update the database
use constant MAX => 16000000; # max size of blob from your MySQL ini - see My.ini : max_allowed_packet = 16M, You may need to set this to 64 MB if you have large assets. My largest was over 10 MB..jeepers!
use constant GETMAX => 1; # set to 1 to query MAX object size (see above line) automatically - much slower to start, but less memory is used overall both at the server and at the client
use constant MANTIS7514 => '/tmp/fix_some_assets.ini'; # path to a temp file to keep track of each run. Delete the file to start at the beginning
# all these modules should already be in core. If not, run "cpan name_of_module', or just go get Strawberry Perl.
use v5.10; # so we can say, and not need to add a newline
use DBI; # database I/O
$|=1; #no buffering STDIO
my $counter = 0; # count of xmlmns corruption
my $havedone = 0;
my $OddCounter = 0; # other corrupt data
mkdir 'corrupt'; # save the data in this folder
my $dbh = DBI->connect(
"dbi:mysql:dbname=$robustDB;host=$hostname",
$username, $password,{RaiseError => 1 , LongReadLen => MAX, LongTruncOk => 0,AutoCommit => 1},
) or die $DBI::errstr;
# Get the largest blob in table data in the DB and set our size of memory buffers to that.
if (GETMAX) {
my $max_len = GetLenData ();
say "Largest object is $max_len bytes";
$dbh->{LongReadLen} = $max_len;
}
my $todo;
my $objectsList = GetAssetList(); # get a hash of all objects
$todo = scalar @$objectsList if ref $objectsList; # count them
say "Found $todo objects";
for (@$objectsList) {
my $obj_uuid = $_->{id};
my $obj_name = $_->{name};
my $obj_time = $_->{create_time};
$havedone++; # keep track of how many done
my $obj_data = GetAssetData ($obj_uuid); # grab the blob
ValidateXML ($obj_data,$obj_uuid, $obj_name); # repair it and check it
#save where we are at so we can be stopped and run again some other rainy day
open(my $in_file, ">",MANTIS7514) || die 'Cannot save the last file date';
print $in_file $obj_time;
close $in_file;
}
say "Found $counter corrupt xmlns objects";
say "Found $OddCounter other possibly corrupted objects";
say "Done!";
sub GetAssetList {
my $answref;
my $time = 0;
# read the last run date from a file
if (open(my $in_file, "<",MANTIS7514))
{
$time = <$in_file> || 0;
close $in_file;
}
my $query = "SELECT id,name, create_time FROM $robustDB.assets WHERE assetType=6 and create_time > $time order by create_time asc;";
$answref = $dbh->selectall_arrayref ($query, { Slice => {} });
return $answref;
}
sub GetAssetData {
my $uuid = shift;
my $query = "SELECT data FROM $robustDB.assets WHERE id='$uuid';";
my $answr = $dbh->selectall_arrayref ($query, { Slice => {} });
return @$answr[0]->{data};
}
sub UpdateData {
return unless UPDATE;
my $uuid = shift;
my $data = shift;
my $sth = $dbh->prepare("UPDATE $robustDB.assets set data = ? WHERE id = ?;");
$sth->execute($data, $uuid) or die $DBI::errstr;;
}
sub GetLenData {
my $len = $dbh->selectrow_array("SELECT MAX(OCTET_LENGTH(data)) FROM $robustDB.assets WHERE assetType=6;");
}
sub ValidateXML {
my $data = shift;
my $obj_uuid = shift;
my $obj_name = shift || '';
#### FORCES AN ERROR FOR DEBUG $data =~ s/xmlns:/xmlns:xmlns:/g; # the fix is in !!!!!!!!!
# Test for repeated xmlns, clever RegEx by Jeff Kelley.
my $corrupt = ($data =~ m/((xmlns:){2,}+)/g);
my $err = $1 || '';
# show them where we are at, if enabled or corrupted.
printf "%d/%d | %s | %s | %s\n", $havedone, $todo, $corrupt ? 'Bad ' : 'Ok ', $obj_uuid, $obj_name if ($corrupt || LISTALL);
if ($corrupt)
{
$counter++;
print ("Found $err\n", 'red') ;
my $original = $data; # so we can save it to disk later, if need be.
$data =~ s/(xmlns:){2,}+/xmlns:/g; # the fix is in
UpdateData($obj_uuid,$data) if UPDATE; # we update if we are enabled and had an error Rev C
save({
name => "$obj_uuid-$obj_name-before.txt",
uuid => $obj_uuid,
corrupt => $err,
data => $original,
type => 0,
});
save({
name => "$obj_uuid-$obj_name-after.txt",
uuid => $obj_uuid,
corrupt => $err,
data => $data,
type => 0,
});
}
return $corrupt; # in case the caller wants to know
}
sub save {
my $c = shift;
$c->{name} =~ s/[^A-Za-z0-9\-\.]//g; # make a safe file name
# may have to force an UTF-16LE or UTF-BE encoding here but my tests show the data is always UTF-8 or ASCII.
if (open(my $out_file, "> :encoding(UTF-8)",'corrupt/' . $c->{name})) {
binmode ($out_file,":encoding(UTF-8)");
$c->{data} =~ s/encoding="utf-8"/encoding="utf-16"/; # not right, but that's they way they coded the XML in the DB.
print $out_file $c->{data};
print $out_file "\0"; # zero termination is in the OAR, lets put it back
} else {
say "Failed to open file";
die;
}
}
__END__
BSD License:
Copyright (c) 2015
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Space here is intentionally left blank for note taking
|
