Add asset fixing Perl script, though likely to convert it to Lua later.

1 files changed, 227 insertions, 0 deletions

diff --git a/scripts/fix_some_assets.pl b/scripts/fix_some_assets.pl
new file mode 100755
index 0000000..08fdcb2
--- /dev/null
+++ b/scripts/fix_some_assets.pl
@@ -0,0 +1,227 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+
+
+# Took almost two and a half hours on IG.
+
+
+=pod
+# 7/30/2015 3:58PM
+Rev B - cleanup
+Rev C - added in missing UPDATE
+
+ quthor Jeff Kelley <opensim@pescadoo.net>
+ mods by Fred Beckhusen <fred@mitsi.com> aka Ferd Frederix
+
+ This version has minimum Perl module dependencies.
+
+ This script connects to an Opensim asset database and scans and fixed bug Mantis-7514
+ The files that are fixed have multiple xmlns : in it, such as xmlns:xmlns:
+
+ http://opensimulator.org/mantis/view.php?id=7514
+
+############################################################################
+# DO A BACKUP BEFORE YOU ATTEMPT THIS. DO A DRY RUN WITH UPDATE SET TO ZERO #
+#############################################################################
+
+ Set variable UPDATE to a 1 to write updates to the db.  A 0 will show you what it would change and also save the bad
+ data to disk in a folder name 'corrupt' for you to examine.
+
+ For safety, we will never update a record unless the UPDATE flag is on.
+
+#############################################################
+# DO NOT RUN THIS UNTIL YOU SET THE CONSTANTS BELOW         #
+#############################################################
+
+=cut
+
+# these are typically found in StandaloneCommon.ini or GridCommon.ini
+
+my $username = '*****';
+my $password = '*****';
+my $hostname = '*****'; # probably you use localhost for standalone, but it could be an IP address or a DNS name for a asset server.
+my $robustDB = '*****'; # your MySQl DB name
+
+# for safety, we will never update a record unless the UPDATE flag is on.
+use constant LISTALL    => 0;                  # show all assets as they are scanned
+use constant UPDATE     => 0;                  # set to 1 if you want to actually update the database
+use constant MAX        => 16000000;           # max size of blob from your MySQL ini - see My.ini :  max_allowed_packet = 16M, You may need to set this to 64 MB if you have large assets. My largest was over 10 MB..jeepers!
+use constant GETMAX     => 1;                  # set to 1 to query MAX object size (see above line) automatically - much slower to start, but less memory is used overall both at the server and at the client
+use constant MANTIS7514 => '/tmp/fix_some_assets.ini';   # path to a temp file to keep track of each run. Delete the file to start at the beginning
+
+# all these modules should already be in core.  If not, run "cpan name_of_module', or just go get Strawberry Perl.
+use v5.10;                    # so we can say, and not need to add a newline
+use DBI;                      # database I/O
+$|=1;  #no buffering STDIO
+
+my $counter = 0;       # count of xmlmns corruption
+my $havedone = 0;
+my $OddCounter = 0;    # other corrupt data
+
+mkdir 'corrupt';       # save the data in this folder
+
+my $dbh = DBI->connect(
+    "dbi:mysql:dbname=$robustDB;host=$hostname",
+    $username, $password,{RaiseError => 1 , LongReadLen => MAX, LongTruncOk => 0,AutoCommit => 1},
+    ) or die $DBI::errstr;
+
+# Get the largest blob in table data in the DB and set our size of memory buffers to that.
+if (GETMAX) {
+    my $max_len = GetLenData ();
+    say "Largest object is $max_len bytes";
+    $dbh->{LongReadLen}  = $max_len;
+}
+
+my $todo;
+my $objectsList = GetAssetList();  # get a hash of all objects
+$todo = scalar @$objectsList if ref $objectsList;   # count them
+say "Found $todo objects";
+
+for (@$objectsList) {
+    my $obj_uuid = $_->{id};
+    my $obj_name = $_->{name};
+    my $obj_time = $_->{create_time};
+
+    $havedone++;   # keep track of how many done
+
+    my $obj_data = GetAssetData ($obj_uuid); # grab the blob
+
+    ValidateXML ($obj_data,$obj_uuid, $obj_name);  # repair it and check it
+
+    #save where we are at so we can be stopped and run again some other rainy day
+    open(my $in_file, ">",MANTIS7514) || die 'Cannot save the last file date';
+    print $in_file $obj_time;
+    close $in_file;
+}
+
+say "Found $counter corrupt xmlns objects";
+say "Found $OddCounter other possibly corrupted objects";
+say "Done!";
+
+
+sub GetAssetList {
+
+    my $answref;
+
+        my $time = 0;
+        # read the last run date from a file
+        if (open(my $in_file, "<",MANTIS7514))
+        {
+           $time = <$in_file> || 0;
+           close $in_file;
+        }
+        my $query = "SELECT id,name, create_time FROM $robustDB.assets WHERE assetType=6 and create_time > $time order by create_time asc;";
+        $answref = $dbh->selectall_arrayref ($query, { Slice => {} });
+
+    return $answref;
+}
+
+sub GetAssetData {
+    my $uuid = shift;
+    my $query = "SELECT data FROM $robustDB.assets WHERE id='$uuid';";
+    my $answr = $dbh->selectall_arrayref ($query, { Slice => {} });
+    return @$answr[0]->{data};
+}
+
+sub UpdateData {
+    return unless UPDATE;
+    my $uuid = shift;
+    my $data = shift;
+
+    my $sth = $dbh->prepare("UPDATE $robustDB.assets set data = ? WHERE id = ?;");
+    $sth->execute($data, $uuid) or die $DBI::errstr;;
+}
+sub GetLenData {
+    my $len = $dbh->selectrow_array("SELECT MAX(OCTET_LENGTH(data)) FROM $robustDB.assets WHERE assetType=6;");
+}
+
+
+sub ValidateXML {
+    my $data = shift;
+    my $obj_uuid = shift;
+    my $obj_name = shift || '';
+
+        ####  FORCES AN ERROR FOR DEBUG $data =~ s/xmlns:/xmlns:xmlns:/g; # the fix is in !!!!!!!!!
+
+    # Test for repeated xmlns, clever RegEx by Jeff Kelley.
+    my $corrupt = ($data =~ m/((xmlns:){2,}+)/g);
+    my $err = $1 || '';
+
+    # show them where we are at, if enabled or corrupted.
+    printf "%d/%d | %s | %s | %s\n", $havedone, $todo, $corrupt ? 'Bad ' : 'Ok ', $obj_uuid, $obj_name if ($corrupt || LISTALL);
+
+    if ($corrupt)
+    {
+        $counter++;
+
+        print ("Found $err\n", 'red') ;
+        my $original = $data;   # so we can save it to disk later, if need be.
+
+        $data =~ s/(xmlns:){2,}+/xmlns:/g; # the fix is in
+
+        UpdateData($obj_uuid,$data) if UPDATE;  # we update if we are enabled and had an error Rev C
+
+        save({
+              name        => "$obj_uuid-$obj_name-before.txt",
+              uuid        => $obj_uuid,
+              corrupt     => $err,
+              data        => $original,
+              type        => 0,
+            });
+
+        save({
+              name        => "$obj_uuid-$obj_name-after.txt",
+              uuid        => $obj_uuid,
+              corrupt     => $err,
+              data        => $data,
+              type        => 0,
+            });
+        }
+
+        return $corrupt; # in case the caller wants to know
+}
+
+sub save {
+    my $c = shift;
+
+        $c->{name} =~ s/[^A-Za-z0-9\-\.]//g; # make a safe file name
+
+        # may have to force an UTF-16LE or UTF-BE encoding here but my tests show the data is always UTF-8 or ASCII.
+
+    if (open(my $out_file, "> :encoding(UTF-8)",'corrupt/' . $c->{name})) {
+        binmode ($out_file,":encoding(UTF-8)");
+
+        $c->{data} =~ s/encoding="utf-8"/encoding="utf-16"/;  # not right, but that's they way they coded the XML in the DB.
+
+        print $out_file $c->{data};
+        print $out_file "\0";       # zero termination is in the OAR, lets put it back
+    } else {
+        say "Failed to open file";
+        die;
+    }
+
+}
+
+__END__
+
+BSD License:
+Copyright (c) 2015
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+Space here is intentionally left blank for note taking