aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/OpenSim
diff options
context:
space:
mode:
authorJustin Clarke Casey2009-05-14 20:37:54 +0000
committerJustin Clarke Casey2009-05-14 20:37:54 +0000
commit62771560448edbc2112d427acd4b37780dfe2154 (patch)
treeba07c1ae1d8de5e24e24a013fcaa39ade898f039 /OpenSim
parent* refactor: move SceneXmlLoader into subpackage (diff)
downloadopensim-SC_OLD-62771560448edbc2112d427acd4b37780dfe2154.zip
opensim-SC_OLD-62771560448edbc2112d427acd4b37780dfe2154.tar.gz
opensim-SC_OLD-62771560448edbc2112d427acd4b37780dfe2154.tar.bz2
opensim-SC_OLD-62771560448edbc2112d427acd4b37780dfe2154.tar.xz
* When saving an oar, save assets when immediately received rather than storing them all up in memory
* Hopefully this will remove out of memory problems when saving large oars on machines without much memory * It may also speed up saving of large oars
Diffstat (limited to 'OpenSim')
-rw-r--r--OpenSim/Framework/Serialization/TarArchiveWriter.cs39
-rw-r--r--OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs21
-rw-r--r--OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs29
-rw-r--r--OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs11
-rw-r--r--OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs150
-rw-r--r--OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs53
6 files changed, 155 insertions, 148 deletions
diff --git a/OpenSim/Framework/Serialization/TarArchiveWriter.cs b/OpenSim/Framework/Serialization/TarArchiveWriter.cs
index d319b0b..316a8de 100644
--- a/OpenSim/Framework/Serialization/TarArchiveWriter.cs
+++ b/OpenSim/Framework/Serialization/TarArchiveWriter.cs
@@ -109,10 +109,14 @@ namespace OpenSim.Framework.Serialization
109 109
110 // Write two consecutive 0 blocks to end the archive 110 // Write two consecutive 0 blocks to end the archive
111 byte[] finalZeroPadding = new byte[1024]; 111 byte[] finalZeroPadding = new byte[1024];
112 m_bw.Write(finalZeroPadding);
113 112
114 m_bw.Flush(); 113 lock (m_bw)
115 m_bw.Close(); 114 {
115 m_bw.Write(finalZeroPadding);
116
117 m_bw.Flush();
118 m_bw.Close();
119 }
116 } 120 }
117 121
118 public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding) 122 public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
@@ -197,20 +201,23 @@ namespace OpenSim.Framework.Serialization
197 201
198 header[154] = 0; 202 header[154] = 0;
199 203
200 // Write out header 204 lock (m_bw)
201 m_bw.Write(header);
202
203 // Write out data
204 m_bw.Write(data);
205
206 if (data.Length % 512 != 0)
207 { 205 {
208 int paddingRequired = 512 - (data.Length % 512); 206 // Write out header
209 207 m_bw.Write(header);
210 //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); 208
211 209 // Write out data
212 byte[] padding = new byte[paddingRequired]; 210 m_bw.Write(data);
213 m_bw.Write(padding); 211
212 if (data.Length % 512 != 0)
213 {
214 int paddingRequired = 512 - (data.Length % 512);
215
216 //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
217
218 byte[] padding = new byte[paddingRequired];
219 m_bw.Write(padding);
220 }
214 } 221 }
215 } 222 }
216 } 223 }
diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
index b5c0d86..fec2425 100644
--- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
@@ -56,7 +56,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
56 private InventoryArchiverModule m_module; 56 private InventoryArchiverModule m_module;
57 private CachedUserInfo m_userInfo; 57 private CachedUserInfo m_userInfo;
58 private string m_invPath; 58 private string m_invPath;
59 protected TarArchiveWriter m_archive; 59 protected TarArchiveWriter m_archiveWriter;
60 protected UuidGatherer m_assetGatherer; 60 protected UuidGatherer m_assetGatherer;
61 61
62 /// <value> 62 /// <value>
@@ -100,17 +100,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
100 m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache); 100 m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache);
101 } 101 }
102 102
103 protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids) 103 protected void ReceivedAllAssets(ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
104 { 104 {
105 AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
106 assetsArchiver.Archive(m_archive);
107
108 Exception reportedException = null; 105 Exception reportedException = null;
109 bool succeeded = true; 106 bool succeeded = true;
110 107
111 try 108 try
112 { 109 {
113 m_archive.Close(); 110 m_archiveWriter.Close();
114 } 111 }
115 catch (IOException e) 112 catch (IOException e)
116 { 113 {
@@ -133,7 +130,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
133 saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager); 130 saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager);
134 131
135 string serialization = UserInventoryItemSerializer.Serialize(saveItem); 132 string serialization = UserInventoryItemSerializer.Serialize(saveItem);
136 m_archive.WriteFile(filename, serialization); 133 m_archiveWriter.WriteFile(filename, serialization);
137 134
138 m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids); 135 m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids);
139 } 136 }
@@ -156,7 +153,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
156 inventoryFolder.ID); 153 inventoryFolder.ID);
157 154
158 // We need to make sure that we record empty folders 155 // We need to make sure that we record empty folders
159 m_archive.WriteDir(path); 156 m_archiveWriter.WriteDir(path);
160 } 157 }
161 158
162 List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls(); 159 List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
@@ -265,7 +262,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
265 inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath); 262 inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
266 } 263 }
267 264
268 m_archive = new TarArchiveWriter(m_saveStream); 265 m_archiveWriter = new TarArchiveWriter(m_saveStream);
269 266
270 if (null == inventoryFolder) 267 if (null == inventoryFolder)
271 { 268 {
@@ -298,7 +295,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
298 } 295 }
299 296
300 SaveUsers(); 297 SaveUsers();
301 new AssetsRequest(m_assetUuids.Keys, m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute(); 298 new AssetsRequest(
299 new AssetsArchiver(m_archiveWriter), m_assetUuids.Keys,
300 m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute();
302 } 301 }
303 302
304 /// <summary> 303 /// <summary>
@@ -316,7 +315,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
316 315
317 if (creator != null) 316 if (creator != null)
318 { 317 {
319 m_archive.WriteFile( 318 m_archiveWriter.WriteFile(
320 ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml", 319 ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml",
321 UserProfileSerializer.Serialize(creator.UserProfile)); 320 UserProfileSerializer.Serialize(creator.UserProfile));
322 } 321 }
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
index 943d9d1..b167ce2 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
@@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
44 /// <summary> 44 /// <summary>
45 /// Method called when all the necessary assets for an archive request have been received. 45 /// Method called when all the necessary assets for an archive request have been received.
46 /// </summary> 46 /// </summary>
47 public delegate void AssetsRequestCallback(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids); 47 public delegate void AssetsRequestCallback(
48 ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids);
48 49
49 /// <summary> 50 /// <summary>
50 /// Execute the write of an archive once we have received all the necessary data 51 /// Execute the write of an archive once we have received all the necessary data
@@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
57 protected IRegionSerialiserModule m_serialiser; 58 protected IRegionSerialiserModule m_serialiser;
58 protected List<SceneObjectGroup> m_sceneObjects; 59 protected List<SceneObjectGroup> m_sceneObjects;
59 protected Scene m_scene; 60 protected Scene m_scene;
60 protected Stream m_saveStream; 61 protected TarArchiveWriter m_archiveWriter;
61 protected Guid m_requestId; 62 protected Guid m_requestId;
62 63
63 public ArchiveWriteRequestExecution( 64 public ArchiveWriteRequestExecution(
@@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
65 ITerrainModule terrainModule, 66 ITerrainModule terrainModule,
66 IRegionSerialiserModule serialiser, 67 IRegionSerialiserModule serialiser,
67 Scene scene, 68 Scene scene,
68 Stream saveStream, 69 TarArchiveWriter archiveWriter,
69 Guid requestId) 70 Guid requestId)
70 { 71 {
71 m_sceneObjects = sceneObjects; 72 m_sceneObjects = sceneObjects;
72 m_terrainModule = terrainModule; 73 m_terrainModule = terrainModule;
73 m_serialiser = serialiser; 74 m_serialiser = serialiser;
74 m_scene = scene; 75 m_scene = scene;
75 m_saveStream = saveStream; 76 m_archiveWriter = archiveWriter;
76 m_requestId = requestId; 77 m_requestId = requestId;
77 } 78 }
78 79
79 protected internal void ReceivedAllAssets( 80 protected internal void ReceivedAllAssets(
80 IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids) 81 ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids)
81 { 82 {
82 foreach (UUID uuid in assetsNotFoundUuids) 83 foreach (UUID uuid in assetsNotFoundUuids)
83 { 84 {
@@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver
86 87
87 m_log.InfoFormat( 88 m_log.InfoFormat(
88 "[ARCHIVER]: Received {0} of {1} assets requested", 89 "[ARCHIVER]: Received {0} of {1} assets requested",
89 assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count); 90 assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count);
90 91
91 m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); 92 m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time.");
92 93
93 TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
94
95 // Write out control file 94 // Write out control file
96 archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); 95 m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
97 96
98 m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); 97 m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
99 98
100 // Write out region settings 99 // Write out region settings
101 string settingsPath 100 string settingsPath
102 = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); 101 = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
103 archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); 102 m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
104 103
105 m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); 104 m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
106 105
@@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
110 109
111 MemoryStream ms = new MemoryStream(); 110 MemoryStream ms = new MemoryStream();
112 m_terrainModule.SaveToStream(terrainPath, ms); 111 m_terrainModule.SaveToStream(terrainPath, ms);
113 archive.WriteFile(terrainPath, ms.ToArray()); 112 m_archiveWriter.WriteFile(terrainPath, ms.ToArray());
114 ms.Close(); 113 ms.Close();
115 114
116 m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); 115 m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
@@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
130 Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), 129 Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
131 sceneObject.UUID); 130 sceneObject.UUID);
132 131
133 archive.WriteFile(filename, serializedObject); 132 m_archiveWriter.WriteFile(filename, serializedObject);
134 } 133 }
135 134
136 m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); 135 m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
137 136
138 // Write out assets 137 m_archiveWriter.Close();
139 AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
140 assetsArchiver.Archive(archive);
141
142 archive.Close();
143 138
144 m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); 139 m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);
145 140
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
index a6ad24c..0a882eb 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs
@@ -35,6 +35,7 @@ using System.Threading;
35using log4net; 35using log4net;
36using OpenMetaverse; 36using OpenMetaverse;
37using OpenSim.Framework; 37using OpenSim.Framework;
38using OpenSim.Framework.Serialization;
38using OpenSim.Region.CoreModules.World.Terrain; 39using OpenSim.Region.CoreModules.World.Terrain;
39using OpenSim.Region.Framework.Interfaces; 40using OpenSim.Region.Framework.Interfaces;
40using OpenSim.Region.Framework.Scenes; 41using OpenSim.Region.Framework.Scenes;
@@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
126 if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) 127 if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4)
127 assetUuids[regionSettings.TerrainTexture4] = 1; 128 assetUuids[regionSettings.TerrainTexture4] = 1;
128 129
130 TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream);
131
129 // Asynchronously request all the assets required to perform this archive operation 132 // Asynchronously request all the assets required to perform this archive operation
130 ArchiveWriteRequestExecution awre 133 ArchiveWriteRequestExecution awre
131 = new ArchiveWriteRequestExecution( 134 = new ArchiveWriteRequestExecution(
@@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
133 m_scene.RequestModuleInterface<ITerrainModule>(), 136 m_scene.RequestModuleInterface<ITerrainModule>(),
134 m_scene.RequestModuleInterface<IRegionSerialiserModule>(), 137 m_scene.RequestModuleInterface<IRegionSerialiserModule>(),
135 m_scene, 138 m_scene,
136 m_saveStream, 139 archiveWriter,
137 m_requestId); 140 m_requestId);
138 141
139 new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); 142 new AssetsRequest(
143 new AssetsArchiver(archiveWriter), assetUuids.Keys,
144 m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute();
140 } 145 }
141 } 146 }
142} 147}
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
index 29d7c5e..0c01cae 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
@@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver
46 /// <value> 46 /// <value>
47 /// Post a message to the log every x assets as a progress bar 47 /// Post a message to the log every x assets as a progress bar
48 /// </value> 48 /// </value>
49 private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; 49 protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50;
50 50
51 /// <summary> 51 /// <value>
52 /// Archive assets 52 /// Keep a count of the number of assets written so that we can provide status updates
53 /// </summary> 53 /// </value>
54 protected IDictionary<UUID, AssetBase> m_assets; 54 protected int m_assetsWritten;
55
56 protected TarArchiveWriter m_archiveWriter;
55 57
56 public AssetsArchiver(IDictionary<UUID, AssetBase> assets) 58 public AssetsArchiver(TarArchiveWriter archiveWriter)
57 { 59 {
58 m_assets = assets; 60 m_archiveWriter = archiveWriter;
59 } 61 }
60 62
61 /// <summary> 63 /// <summary>
62 /// Archive the assets given to this archiver to the given archive. 64 /// Archive the assets given to this archiver to the given archive.
63 /// </summary> 65 /// </summary>
64 /// <param name="archive"></param> 66 /// <param name="archive"></param>
65 public void Archive(TarArchiveWriter archive) 67 public void WriteAsset(AssetBase asset)
66 { 68 {
67 //WriteMetadata(archive); 69 //WriteMetadata(archive);
68 WriteData(archive); 70 WriteData(asset);
69 } 71 }
70 72
71 /// <summary> 73 /// <summary>
72 /// Write an assets metadata file to the given archive 74 /// Write an assets metadata file to the given archive
73 /// </summary> 75 /// </summary>
74 /// <param name="archive"></param> 76 /// <param name="archive"></param>
75 protected void WriteMetadata(TarArchiveWriter archive) 77// protected void WriteMetadata(TarArchiveWriter archive)
76 { 78// {
77 StringWriter sw = new StringWriter(); 79// StringWriter sw = new StringWriter();
78 XmlTextWriter xtw = new XmlTextWriter(sw); 80// XmlTextWriter xtw = new XmlTextWriter(sw);
79 81//
80 xtw.Formatting = Formatting.Indented; 82// xtw.Formatting = Formatting.Indented;
81 xtw.WriteStartDocument(); 83// xtw.WriteStartDocument();
82 84//
83 xtw.WriteStartElement("assets"); 85// xtw.WriteStartElement("assets");
84 86//
85 foreach (UUID uuid in m_assets.Keys) 87// foreach (UUID uuid in m_assets.Keys)
86 { 88// {
87 AssetBase asset = m_assets[uuid]; 89// AssetBase asset = m_assets[uuid];
88 90//
89 if (asset != null) 91// if (asset != null)
90 { 92// {
91 xtw.WriteStartElement("asset"); 93// xtw.WriteStartElement("asset");
92 94//
93 string extension = string.Empty; 95// string extension = string.Empty;
94 96//
95 if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) 97// if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
96 { 98// {
97 extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; 99// extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
98 } 100// }
99 101//
100 xtw.WriteElementString("filename", uuid.ToString() + extension); 102// xtw.WriteElementString("filename", uuid.ToString() + extension);
101 103//
102 xtw.WriteElementString("name", asset.Name); 104// xtw.WriteElementString("name", asset.Name);
103 xtw.WriteElementString("description", asset.Description); 105// xtw.WriteElementString("description", asset.Description);
104 xtw.WriteElementString("asset-type", asset.Type.ToString()); 106// xtw.WriteElementString("asset-type", asset.Type.ToString());
105 107//
106 xtw.WriteEndElement(); 108// xtw.WriteEndElement();
107 } 109// }
108 } 110// }
109 111//
110 xtw.WriteEndElement(); 112// xtw.WriteEndElement();
111 113//
112 xtw.WriteEndDocument(); 114// xtw.WriteEndDocument();
113 115//
114 archive.WriteFile("assets.xml", sw.ToString()); 116// archive.WriteFile("assets.xml", sw.ToString());
115 } 117// }
116 118
117 /// <summary> 119 /// <summary>
118 /// Write asset data files to the given archive 120 /// Write asset data files to the given archive
119 /// </summary> 121 /// </summary>
120 /// <param name="archive"></param> 122 /// <param name="asset"></param>
121 protected void WriteData(TarArchiveWriter archive) 123 protected void WriteData(AssetBase asset)
122 { 124 {
123 // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar 125 // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar
124 //archive.AddDir("assets"); 126 //archive.AddDir("assets");
125 127
126 int assetsAdded = 0; 128 string extension = string.Empty;
127 129
128 foreach (UUID uuid in m_assets.Keys) 130 if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type))
129 { 131 {
130 AssetBase asset = m_assets[uuid]; 132 extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type];
131 133 }
132 string extension = string.Empty; 134 else
133 135 {
134 if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) 136 m_log.ErrorFormat(
135 { 137 "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded",
136 extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; 138 asset.Type, asset.ID);
137 } 139 }
138 else
139 {
140 m_log.ErrorFormat(
141 "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded",
142 asset.Type, asset.ID);
143 }
144
145 archive.WriteFile(
146 ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
147 asset.Data);
148 140
149 assetsAdded++; 141 m_archiveWriter.WriteFile(
142 ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension,
143 asset.Data);
150 144
151 if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) 145 m_assetsWritten++;
152 m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded);
153 }
154 146
155 if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0) 147 if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0)
156 m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); 148 m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten);
157 } 149 }
158 } 150 }
159} 151}
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index 0a0bb4c..d806a9c 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -32,6 +32,7 @@ using System.Threading;
32using log4net; 32using log4net;
33using OpenMetaverse; 33using OpenMetaverse;
34using OpenSim.Framework; 34using OpenSim.Framework;
35using OpenSim.Framework.Serialization;
35 36
36namespace OpenSim.Region.CoreModules.World.Archiver 37namespace OpenSim.Region.CoreModules.World.Archiver
37{ 38{
@@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver
42 { 43 {
43 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); 44 private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
44 45
45 /// <summary> 46 /// <value>
46 /// uuids to request 47 /// uuids to request
47 /// </summary> 48 /// </value>
48 protected ICollection<UUID> m_uuids; 49 protected ICollection<UUID> m_uuids;
49 50
50 /// <summary> 51 /// <value>
51 /// Callback used when all the assets requested have been received. 52 /// Callback used when all the assets requested have been received.
52 /// </summary> 53 /// </value>
53 protected AssetsRequestCallback m_assetsRequestCallback; 54 protected AssetsRequestCallback m_assetsRequestCallback;
54 55
55 /// <summary> 56 /// <value>
56 /// Assets retrieved in this request 57 /// List of assets that were found. This will be passed back to the requester.
57 /// </summary> 58 /// </value>
58 protected Dictionary<UUID, AssetBase> m_assets = new Dictionary<UUID, AssetBase>(); 59 protected List<UUID> m_foundAssetUuids = new List<UUID>();
59 60
60 /// <summary> 61 /// <value>
61 /// Maintain a list of assets that could not be found. This will be passed back to the requester. 62 /// Maintain a list of assets that could not be found. This will be passed back to the requester.
62 /// </summary> 63 /// </value>
63 protected List<UUID> m_notFoundAssetUuids = new List<UUID>(); 64 protected List<UUID> m_notFoundAssetUuids = new List<UUID>();
64 65
65 /// <summary> 66 /// <value>
66 /// Record the number of asset replies required so we know when we've finished 67 /// Record the number of asset replies required so we know when we've finished
67 /// </summary> 68 /// </value>
68 private int m_repliesRequired; 69 private int m_repliesRequired;
69 70
70 /// <summary> 71 /// <value>
71 /// Asset cache used to request the assets 72 /// Asset cache used to request the assets
72 /// </summary> 73 /// </value>
73 protected IAssetCache m_assetCache; 74 protected IAssetCache m_assetCache;
74 75
76 protected AssetsArchiver m_assetsArchiver;
77
75 protected internal AssetsRequest( 78 protected internal AssetsRequest(
76 ICollection<UUID> uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) 79 AssetsArchiver assetsArchiver, ICollection<UUID> uuids,
80 IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback)
77 { 81 {
82 m_assetsArchiver = assetsArchiver;
78 m_uuids = uuids; 83 m_uuids = uuids;
79 m_assetsRequestCallback = assetsRequestCallback; 84 m_assetsRequestCallback = assetsRequestCallback;
80 m_assetCache = assetCache; 85 m_assetCache = assetCache;
@@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
87 92
88 // We can stop here if there are no assets to fetch 93 // We can stop here if there are no assets to fetch
89 if (m_repliesRequired == 0) 94 if (m_repliesRequired == 0)
90 m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); 95 m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
91 96
92 foreach (UUID uuid in m_uuids) 97 foreach (UUID uuid in m_uuids)
93 { 98 {
@@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver
106 111
107 if (asset != null) 112 if (asset != null)
108 { 113 {
114 // Make sure that we don't run out of memory by hogging assets in the cache
109 m_assetCache.ExpireAsset(assetID); 115 m_assetCache.ExpireAsset(assetID);
110 m_assets[assetID] = asset; 116
117 m_foundAssetUuids.Add(assetID);
118 m_assetsArchiver.WriteAsset(asset);
111 } 119 }
112 else 120 else
113 { 121 {
114 m_notFoundAssetUuids.Add(assetID); 122 m_notFoundAssetUuids.Add(assetID);
115 } 123 }
116 124
117 if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired) 125 if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired)
118 { 126 {
119 m_log.DebugFormat( 127 m_log.DebugFormat(
120 "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets", 128 "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets",
121 m_assets.Count, m_notFoundAssetUuids.Count); 129 m_foundAssetUuids.Count, m_notFoundAssetUuids.Count);
122 130
123 // We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive 131 // We want to stop using the asset cache thread asap
132 // as we now need to do the work of producing the rest of the archive
124 Thread newThread = new Thread(PerformAssetsRequestCallback); 133 Thread newThread = new Thread(PerformAssetsRequestCallback);
125 newThread.Name = "OpenSimulator archiving thread post assets receipt"; 134 newThread.Name = "OpenSimulator archiving thread post assets receipt";
126 newThread.Start(); 135 newThread.Start();
@@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
134 { 143 {
135 try 144 try
136 { 145 {
137 m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); 146 m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids);
138 } 147 }
139 catch (Exception e) 148 catch (Exception e)
140 { 149 {