diff options
author | Justin Clarke Casey | 2009-05-14 20:37:54 +0000 |
---|---|---|
committer | Justin Clarke Casey | 2009-05-14 20:37:54 +0000 |
commit | 62771560448edbc2112d427acd4b37780dfe2154 (patch) | |
tree | ba07c1ae1d8de5e24e24a013fcaa39ade898f039 | |
parent | * refactor: move SceneXmlLoader into subpackage (diff) | |
download | opensim-SC-62771560448edbc2112d427acd4b37780dfe2154.zip opensim-SC-62771560448edbc2112d427acd4b37780dfe2154.tar.gz opensim-SC-62771560448edbc2112d427acd4b37780dfe2154.tar.bz2 opensim-SC-62771560448edbc2112d427acd4b37780dfe2154.tar.xz |
* When saving an oar, save assets when immediately received rather than storing them all up in memory
* Hopefully this will remove out of memory problems when saving large oars on machines without much memory
* It may also speed up saving of large oars
6 files changed, 155 insertions, 148 deletions
diff --git a/OpenSim/Framework/Serialization/TarArchiveWriter.cs b/OpenSim/Framework/Serialization/TarArchiveWriter.cs index d319b0b..316a8de 100644 --- a/OpenSim/Framework/Serialization/TarArchiveWriter.cs +++ b/OpenSim/Framework/Serialization/TarArchiveWriter.cs | |||
@@ -109,10 +109,14 @@ namespace OpenSim.Framework.Serialization | |||
109 | 109 | ||
110 | // Write two consecutive 0 blocks to end the archive | 110 | // Write two consecutive 0 blocks to end the archive |
111 | byte[] finalZeroPadding = new byte[1024]; | 111 | byte[] finalZeroPadding = new byte[1024]; |
112 | m_bw.Write(finalZeroPadding); | ||
113 | 112 | ||
114 | m_bw.Flush(); | 113 | lock (m_bw) |
115 | m_bw.Close(); | 114 | { |
115 | m_bw.Write(finalZeroPadding); | ||
116 | |||
117 | m_bw.Flush(); | ||
118 | m_bw.Close(); | ||
119 | } | ||
116 | } | 120 | } |
117 | 121 | ||
118 | public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding) | 122 | public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding) |
@@ -197,20 +201,23 @@ namespace OpenSim.Framework.Serialization | |||
197 | 201 | ||
198 | header[154] = 0; | 202 | header[154] = 0; |
199 | 203 | ||
200 | // Write out header | 204 | lock (m_bw) |
201 | m_bw.Write(header); | ||
202 | |||
203 | // Write out data | ||
204 | m_bw.Write(data); | ||
205 | |||
206 | if (data.Length % 512 != 0) | ||
207 | { | 205 | { |
208 | int paddingRequired = 512 - (data.Length % 512); | 206 | // Write out header |
209 | 207 | m_bw.Write(header); | |
210 | //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); | 208 | |
211 | 209 | // Write out data | |
212 | byte[] padding = new byte[paddingRequired]; | 210 | m_bw.Write(data); |
213 | m_bw.Write(padding); | 211 | |
212 | if (data.Length % 512 != 0) | ||
213 | { | ||
214 | int paddingRequired = 512 - (data.Length % 512); | ||
215 | |||
216 | //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired); | ||
217 | |||
218 | byte[] padding = new byte[paddingRequired]; | ||
219 | m_bw.Write(padding); | ||
220 | } | ||
214 | } | 221 | } |
215 | } | 222 | } |
216 | } | 223 | } |
diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs index b5c0d86..fec2425 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs | |||
@@ -56,7 +56,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
56 | private InventoryArchiverModule m_module; | 56 | private InventoryArchiverModule m_module; |
57 | private CachedUserInfo m_userInfo; | 57 | private CachedUserInfo m_userInfo; |
58 | private string m_invPath; | 58 | private string m_invPath; |
59 | protected TarArchiveWriter m_archive; | 59 | protected TarArchiveWriter m_archiveWriter; |
60 | protected UuidGatherer m_assetGatherer; | 60 | protected UuidGatherer m_assetGatherer; |
61 | 61 | ||
62 | /// <value> | 62 | /// <value> |
@@ -100,17 +100,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
100 | m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache); | 100 | m_assetGatherer = new UuidGatherer(m_module.CommsManager.AssetCache); |
101 | } | 101 | } |
102 | 102 | ||
103 | protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids) | 103 | protected void ReceivedAllAssets(ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids) |
104 | { | 104 | { |
105 | AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); | ||
106 | assetsArchiver.Archive(m_archive); | ||
107 | |||
108 | Exception reportedException = null; | 105 | Exception reportedException = null; |
109 | bool succeeded = true; | 106 | bool succeeded = true; |
110 | 107 | ||
111 | try | 108 | try |
112 | { | 109 | { |
113 | m_archive.Close(); | 110 | m_archiveWriter.Close(); |
114 | } | 111 | } |
115 | catch (IOException e) | 112 | catch (IOException e) |
116 | { | 113 | { |
@@ -133,7 +130,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
133 | saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager); | 130 | saveItem.CreatorId = OspResolver.MakeOspa(saveItem.CreatorIdAsUuid, m_module.CommsManager); |
134 | 131 | ||
135 | string serialization = UserInventoryItemSerializer.Serialize(saveItem); | 132 | string serialization = UserInventoryItemSerializer.Serialize(saveItem); |
136 | m_archive.WriteFile(filename, serialization); | 133 | m_archiveWriter.WriteFile(filename, serialization); |
137 | 134 | ||
138 | m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids); | 135 | m_assetGatherer.GatherAssetUuids(saveItem.AssetID, (AssetType)saveItem.AssetType, m_assetUuids); |
139 | } | 136 | } |
@@ -156,7 +153,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
156 | inventoryFolder.ID); | 153 | inventoryFolder.ID); |
157 | 154 | ||
158 | // We need to make sure that we record empty folders | 155 | // We need to make sure that we record empty folders |
159 | m_archive.WriteDir(path); | 156 | m_archiveWriter.WriteDir(path); |
160 | } | 157 | } |
161 | 158 | ||
162 | List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls(); | 159 | List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls(); |
@@ -265,7 +262,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
265 | inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath); | 262 | inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath); |
266 | } | 263 | } |
267 | 264 | ||
268 | m_archive = new TarArchiveWriter(m_saveStream); | 265 | m_archiveWriter = new TarArchiveWriter(m_saveStream); |
269 | 266 | ||
270 | if (null == inventoryFolder) | 267 | if (null == inventoryFolder) |
271 | { | 268 | { |
@@ -298,7 +295,9 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
298 | } | 295 | } |
299 | 296 | ||
300 | SaveUsers(); | 297 | SaveUsers(); |
301 | new AssetsRequest(m_assetUuids.Keys, m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute(); | 298 | new AssetsRequest( |
299 | new AssetsArchiver(m_archiveWriter), m_assetUuids.Keys, | ||
300 | m_module.CommsManager.AssetCache, ReceivedAllAssets).Execute(); | ||
302 | } | 301 | } |
303 | 302 | ||
304 | /// <summary> | 303 | /// <summary> |
@@ -316,7 +315,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver | |||
316 | 315 | ||
317 | if (creator != null) | 316 | if (creator != null) |
318 | { | 317 | { |
319 | m_archive.WriteFile( | 318 | m_archiveWriter.WriteFile( |
320 | ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml", | 319 | ArchiveConstants.USERS_PATH + creator.UserProfile.Name + ".xml", |
321 | UserProfileSerializer.Serialize(creator.UserProfile)); | 320 | UserProfileSerializer.Serialize(creator.UserProfile)); |
322 | } | 321 | } |
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs index 943d9d1..b167ce2 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs | |||
@@ -44,7 +44,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
44 | /// <summary> | 44 | /// <summary> |
45 | /// Method called when all the necessary assets for an archive request have been received. | 45 | /// Method called when all the necessary assets for an archive request have been received. |
46 | /// </summary> | 46 | /// </summary> |
47 | public delegate void AssetsRequestCallback(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids); | 47 | public delegate void AssetsRequestCallback( |
48 | ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids); | ||
48 | 49 | ||
49 | /// <summary> | 50 | /// <summary> |
50 | /// Execute the write of an archive once we have received all the necessary data | 51 | /// Execute the write of an archive once we have received all the necessary data |
@@ -57,7 +58,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
57 | protected IRegionSerialiserModule m_serialiser; | 58 | protected IRegionSerialiserModule m_serialiser; |
58 | protected List<SceneObjectGroup> m_sceneObjects; | 59 | protected List<SceneObjectGroup> m_sceneObjects; |
59 | protected Scene m_scene; | 60 | protected Scene m_scene; |
60 | protected Stream m_saveStream; | 61 | protected TarArchiveWriter m_archiveWriter; |
61 | protected Guid m_requestId; | 62 | protected Guid m_requestId; |
62 | 63 | ||
63 | public ArchiveWriteRequestExecution( | 64 | public ArchiveWriteRequestExecution( |
@@ -65,19 +66,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
65 | ITerrainModule terrainModule, | 66 | ITerrainModule terrainModule, |
66 | IRegionSerialiserModule serialiser, | 67 | IRegionSerialiserModule serialiser, |
67 | Scene scene, | 68 | Scene scene, |
68 | Stream saveStream, | 69 | TarArchiveWriter archiveWriter, |
69 | Guid requestId) | 70 | Guid requestId) |
70 | { | 71 | { |
71 | m_sceneObjects = sceneObjects; | 72 | m_sceneObjects = sceneObjects; |
72 | m_terrainModule = terrainModule; | 73 | m_terrainModule = terrainModule; |
73 | m_serialiser = serialiser; | 74 | m_serialiser = serialiser; |
74 | m_scene = scene; | 75 | m_scene = scene; |
75 | m_saveStream = saveStream; | 76 | m_archiveWriter = archiveWriter; |
76 | m_requestId = requestId; | 77 | m_requestId = requestId; |
77 | } | 78 | } |
78 | 79 | ||
79 | protected internal void ReceivedAllAssets( | 80 | protected internal void ReceivedAllAssets( |
80 | IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids) | 81 | ICollection<UUID> assetsFoundUuids, ICollection<UUID> assetsNotFoundUuids) |
81 | { | 82 | { |
82 | foreach (UUID uuid in assetsNotFoundUuids) | 83 | foreach (UUID uuid in assetsNotFoundUuids) |
83 | { | 84 | { |
@@ -86,21 +87,19 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
86 | 87 | ||
87 | m_log.InfoFormat( | 88 | m_log.InfoFormat( |
88 | "[ARCHIVER]: Received {0} of {1} assets requested", | 89 | "[ARCHIVER]: Received {0} of {1} assets requested", |
89 | assetsFound.Count, assetsFound.Count + assetsNotFoundUuids.Count); | 90 | assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count); |
90 | 91 | ||
91 | m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); | 92 | m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time."); |
92 | 93 | ||
93 | TarArchiveWriter archive = new TarArchiveWriter(m_saveStream); | ||
94 | |||
95 | // Write out control file | 94 | // Write out control file |
96 | archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); | 95 | m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile()); |
97 | 96 | ||
98 | m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); | 97 | m_log.InfoFormat("[ARCHIVER]: Added control file to archive."); |
99 | 98 | ||
100 | // Write out region settings | 99 | // Write out region settings |
101 | string settingsPath | 100 | string settingsPath |
102 | = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); | 101 | = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName); |
103 | archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); | 102 | m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings)); |
104 | 103 | ||
105 | m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); | 104 | m_log.InfoFormat("[ARCHIVER]: Added region settings to archive."); |
106 | 105 | ||
@@ -110,7 +109,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
110 | 109 | ||
111 | MemoryStream ms = new MemoryStream(); | 110 | MemoryStream ms = new MemoryStream(); |
112 | m_terrainModule.SaveToStream(terrainPath, ms); | 111 | m_terrainModule.SaveToStream(terrainPath, ms); |
113 | archive.WriteFile(terrainPath, ms.ToArray()); | 112 | m_archiveWriter.WriteFile(terrainPath, ms.ToArray()); |
114 | ms.Close(); | 113 | ms.Close(); |
115 | 114 | ||
116 | m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); | 115 | m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive."); |
@@ -130,16 +129,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
130 | Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), | 129 | Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z), |
131 | sceneObject.UUID); | 130 | sceneObject.UUID); |
132 | 131 | ||
133 | archive.WriteFile(filename, serializedObject); | 132 | m_archiveWriter.WriteFile(filename, serializedObject); |
134 | } | 133 | } |
135 | 134 | ||
136 | m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); | 135 | m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive."); |
137 | 136 | ||
138 | // Write out assets | 137 | m_archiveWriter.Close(); |
139 | AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound); | ||
140 | assetsArchiver.Archive(archive); | ||
141 | |||
142 | archive.Close(); | ||
143 | 138 | ||
144 | m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); | 139 | m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName); |
145 | 140 | ||
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs index a6ad24c..0a882eb 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestPreparation.cs | |||
@@ -35,6 +35,7 @@ using System.Threading; | |||
35 | using log4net; | 35 | using log4net; |
36 | using OpenMetaverse; | 36 | using OpenMetaverse; |
37 | using OpenSim.Framework; | 37 | using OpenSim.Framework; |
38 | using OpenSim.Framework.Serialization; | ||
38 | using OpenSim.Region.CoreModules.World.Terrain; | 39 | using OpenSim.Region.CoreModules.World.Terrain; |
39 | using OpenSim.Region.Framework.Interfaces; | 40 | using OpenSim.Region.Framework.Interfaces; |
40 | using OpenSim.Region.Framework.Scenes; | 41 | using OpenSim.Region.Framework.Scenes; |
@@ -126,6 +127,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
126 | if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) | 127 | if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4) |
127 | assetUuids[regionSettings.TerrainTexture4] = 1; | 128 | assetUuids[regionSettings.TerrainTexture4] = 1; |
128 | 129 | ||
130 | TarArchiveWriter archiveWriter = new TarArchiveWriter(m_saveStream); | ||
131 | |||
129 | // Asynchronously request all the assets required to perform this archive operation | 132 | // Asynchronously request all the assets required to perform this archive operation |
130 | ArchiveWriteRequestExecution awre | 133 | ArchiveWriteRequestExecution awre |
131 | = new ArchiveWriteRequestExecution( | 134 | = new ArchiveWriteRequestExecution( |
@@ -133,10 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
133 | m_scene.RequestModuleInterface<ITerrainModule>(), | 136 | m_scene.RequestModuleInterface<ITerrainModule>(), |
134 | m_scene.RequestModuleInterface<IRegionSerialiserModule>(), | 137 | m_scene.RequestModuleInterface<IRegionSerialiserModule>(), |
135 | m_scene, | 138 | m_scene, |
136 | m_saveStream, | 139 | archiveWriter, |
137 | m_requestId); | 140 | m_requestId); |
138 | 141 | ||
139 | new AssetsRequest(assetUuids.Keys, m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); | 142 | new AssetsRequest( |
143 | new AssetsArchiver(archiveWriter), assetUuids.Keys, | ||
144 | m_scene.CommsManager.AssetCache, awre.ReceivedAllAssets).Execute(); | ||
140 | } | 145 | } |
141 | } | 146 | } |
142 | } | 147 | } |
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs index 29d7c5e..0c01cae 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs | |||
@@ -46,114 +46,106 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
46 | /// <value> | 46 | /// <value> |
47 | /// Post a message to the log every x assets as a progress bar | 47 | /// Post a message to the log every x assets as a progress bar |
48 | /// </value> | 48 | /// </value> |
49 | private static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; | 49 | protected static int LOG_ASSET_LOAD_NOTIFICATION_INTERVAL = 50; |
50 | 50 | ||
51 | /// <summary> | 51 | /// <value> |
52 | /// Archive assets | 52 | /// Keep a count of the number of assets written so that we can provide status updates |
53 | /// </summary> | 53 | /// </value> |
54 | protected IDictionary<UUID, AssetBase> m_assets; | 54 | protected int m_assetsWritten; |
55 | |||
56 | protected TarArchiveWriter m_archiveWriter; | ||
55 | 57 | ||
56 | public AssetsArchiver(IDictionary<UUID, AssetBase> assets) | 58 | public AssetsArchiver(TarArchiveWriter archiveWriter) |
57 | { | 59 | { |
58 | m_assets = assets; | 60 | m_archiveWriter = archiveWriter; |
59 | } | 61 | } |
60 | 62 | ||
61 | /// <summary> | 63 | /// <summary> |
62 | /// Archive the assets given to this archiver to the given archive. | 64 | /// Archive the assets given to this archiver to the given archive. |
63 | /// </summary> | 65 | /// </summary> |
64 | /// <param name="archive"></param> | 66 | /// <param name="archive"></param> |
65 | public void Archive(TarArchiveWriter archive) | 67 | public void WriteAsset(AssetBase asset) |
66 | { | 68 | { |
67 | //WriteMetadata(archive); | 69 | //WriteMetadata(archive); |
68 | WriteData(archive); | 70 | WriteData(asset); |
69 | } | 71 | } |
70 | 72 | ||
71 | /// <summary> | 73 | /// <summary> |
72 | /// Write an assets metadata file to the given archive | 74 | /// Write an assets metadata file to the given archive |
73 | /// </summary> | 75 | /// </summary> |
74 | /// <param name="archive"></param> | 76 | /// <param name="archive"></param> |
75 | protected void WriteMetadata(TarArchiveWriter archive) | 77 | // protected void WriteMetadata(TarArchiveWriter archive) |
76 | { | 78 | // { |
77 | StringWriter sw = new StringWriter(); | 79 | // StringWriter sw = new StringWriter(); |
78 | XmlTextWriter xtw = new XmlTextWriter(sw); | 80 | // XmlTextWriter xtw = new XmlTextWriter(sw); |
79 | 81 | // | |
80 | xtw.Formatting = Formatting.Indented; | 82 | // xtw.Formatting = Formatting.Indented; |
81 | xtw.WriteStartDocument(); | 83 | // xtw.WriteStartDocument(); |
82 | 84 | // | |
83 | xtw.WriteStartElement("assets"); | 85 | // xtw.WriteStartElement("assets"); |
84 | 86 | // | |
85 | foreach (UUID uuid in m_assets.Keys) | 87 | // foreach (UUID uuid in m_assets.Keys) |
86 | { | 88 | // { |
87 | AssetBase asset = m_assets[uuid]; | 89 | // AssetBase asset = m_assets[uuid]; |
88 | 90 | // | |
89 | if (asset != null) | 91 | // if (asset != null) |
90 | { | 92 | // { |
91 | xtw.WriteStartElement("asset"); | 93 | // xtw.WriteStartElement("asset"); |
92 | 94 | // | |
93 | string extension = string.Empty; | 95 | // string extension = string.Empty; |
94 | 96 | // | |
95 | if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) | 97 | // if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) |
96 | { | 98 | // { |
97 | extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; | 99 | // extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; |
98 | } | 100 | // } |
99 | 101 | // | |
100 | xtw.WriteElementString("filename", uuid.ToString() + extension); | 102 | // xtw.WriteElementString("filename", uuid.ToString() + extension); |
101 | 103 | // | |
102 | xtw.WriteElementString("name", asset.Name); | 104 | // xtw.WriteElementString("name", asset.Name); |
103 | xtw.WriteElementString("description", asset.Description); | 105 | // xtw.WriteElementString("description", asset.Description); |
104 | xtw.WriteElementString("asset-type", asset.Type.ToString()); | 106 | // xtw.WriteElementString("asset-type", asset.Type.ToString()); |
105 | 107 | // | |
106 | xtw.WriteEndElement(); | 108 | // xtw.WriteEndElement(); |
107 | } | 109 | // } |
108 | } | 110 | // } |
109 | 111 | // | |
110 | xtw.WriteEndElement(); | 112 | // xtw.WriteEndElement(); |
111 | 113 | // | |
112 | xtw.WriteEndDocument(); | 114 | // xtw.WriteEndDocument(); |
113 | 115 | // | |
114 | archive.WriteFile("assets.xml", sw.ToString()); | 116 | // archive.WriteFile("assets.xml", sw.ToString()); |
115 | } | 117 | // } |
116 | 118 | ||
117 | /// <summary> | 119 | /// <summary> |
118 | /// Write asset data files to the given archive | 120 | /// Write asset data files to the given archive |
119 | /// </summary> | 121 | /// </summary> |
120 | /// <param name="archive"></param> | 122 | /// <param name="asset"></param> |
121 | protected void WriteData(TarArchiveWriter archive) | 123 | protected void WriteData(AssetBase asset) |
122 | { | 124 | { |
123 | // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar | 125 | // It appears that gtar, at least, doesn't need the intermediate directory entries in the tar |
124 | //archive.AddDir("assets"); | 126 | //archive.AddDir("assets"); |
125 | 127 | ||
126 | int assetsAdded = 0; | 128 | string extension = string.Empty; |
127 | 129 | ||
128 | foreach (UUID uuid in m_assets.Keys) | 130 | if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) |
129 | { | 131 | { |
130 | AssetBase asset = m_assets[uuid]; | 132 | extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; |
131 | 133 | } | |
132 | string extension = string.Empty; | 134 | else |
133 | 135 | { | |
134 | if (ArchiveConstants.ASSET_TYPE_TO_EXTENSION.ContainsKey(asset.Type)) | 136 | m_log.ErrorFormat( |
135 | { | 137 | "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded", |
136 | extension = ArchiveConstants.ASSET_TYPE_TO_EXTENSION[asset.Type]; | 138 | asset.Type, asset.ID); |
137 | } | 139 | } |
138 | else | ||
139 | { | ||
140 | m_log.ErrorFormat( | ||
141 | "[ARCHIVER]: Unrecognized asset type {0} with uuid {1}. This asset will be saved but not reloaded", | ||
142 | asset.Type, asset.ID); | ||
143 | } | ||
144 | |||
145 | archive.WriteFile( | ||
146 | ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension, | ||
147 | asset.Data); | ||
148 | 140 | ||
149 | assetsAdded++; | 141 | m_archiveWriter.WriteFile( |
142 | ArchiveConstants.ASSETS_PATH + asset.FullID.ToString() + extension, | ||
143 | asset.Data); | ||
150 | 144 | ||
151 | if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) | 145 | m_assetsWritten++; |
152 | m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); | ||
153 | } | ||
154 | 146 | ||
155 | if (assetsAdded % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL != 0) | 147 | if (m_assetsWritten % LOG_ASSET_LOAD_NOTIFICATION_INTERVAL == 0) |
156 | m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", assetsAdded); | 148 | m_log.InfoFormat("[ARCHIVER]: Added {0} assets to archive", m_assetsWritten); |
157 | } | 149 | } |
158 | } | 150 | } |
159 | } | 151 | } |
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index 0a0bb4c..d806a9c 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs | |||
@@ -32,6 +32,7 @@ using System.Threading; | |||
32 | using log4net; | 32 | using log4net; |
33 | using OpenMetaverse; | 33 | using OpenMetaverse; |
34 | using OpenSim.Framework; | 34 | using OpenSim.Framework; |
35 | using OpenSim.Framework.Serialization; | ||
35 | 36 | ||
36 | namespace OpenSim.Region.CoreModules.World.Archiver | 37 | namespace OpenSim.Region.CoreModules.World.Archiver |
37 | { | 38 | { |
@@ -42,39 +43,43 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
42 | { | 43 | { |
43 | private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); | 44 | private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
44 | 45 | ||
45 | /// <summary> | 46 | /// <value> |
46 | /// uuids to request | 47 | /// uuids to request |
47 | /// </summary> | 48 | /// </value> |
48 | protected ICollection<UUID> m_uuids; | 49 | protected ICollection<UUID> m_uuids; |
49 | 50 | ||
50 | /// <summary> | 51 | /// <value> |
51 | /// Callback used when all the assets requested have been received. | 52 | /// Callback used when all the assets requested have been received. |
52 | /// </summary> | 53 | /// </value> |
53 | protected AssetsRequestCallback m_assetsRequestCallback; | 54 | protected AssetsRequestCallback m_assetsRequestCallback; |
54 | 55 | ||
55 | /// <summary> | 56 | /// <value> |
56 | /// Assets retrieved in this request | 57 | /// List of assets that were found. This will be passed back to the requester. |
57 | /// </summary> | 58 | /// </value> |
58 | protected Dictionary<UUID, AssetBase> m_assets = new Dictionary<UUID, AssetBase>(); | 59 | protected List<UUID> m_foundAssetUuids = new List<UUID>(); |
59 | 60 | ||
60 | /// <summary> | 61 | /// <value> |
61 | /// Maintain a list of assets that could not be found. This will be passed back to the requester. | 62 | /// Maintain a list of assets that could not be found. This will be passed back to the requester. |
62 | /// </summary> | 63 | /// </value> |
63 | protected List<UUID> m_notFoundAssetUuids = new List<UUID>(); | 64 | protected List<UUID> m_notFoundAssetUuids = new List<UUID>(); |
64 | 65 | ||
65 | /// <summary> | 66 | /// <value> |
66 | /// Record the number of asset replies required so we know when we've finished | 67 | /// Record the number of asset replies required so we know when we've finished |
67 | /// </summary> | 68 | /// </value> |
68 | private int m_repliesRequired; | 69 | private int m_repliesRequired; |
69 | 70 | ||
70 | /// <summary> | 71 | /// <value> |
71 | /// Asset cache used to request the assets | 72 | /// Asset cache used to request the assets |
72 | /// </summary> | 73 | /// </value> |
73 | protected IAssetCache m_assetCache; | 74 | protected IAssetCache m_assetCache; |
74 | 75 | ||
76 | protected AssetsArchiver m_assetsArchiver; | ||
77 | |||
75 | protected internal AssetsRequest( | 78 | protected internal AssetsRequest( |
76 | ICollection<UUID> uuids, IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) | 79 | AssetsArchiver assetsArchiver, ICollection<UUID> uuids, |
80 | IAssetCache assetCache, AssetsRequestCallback assetsRequestCallback) | ||
77 | { | 81 | { |
82 | m_assetsArchiver = assetsArchiver; | ||
78 | m_uuids = uuids; | 83 | m_uuids = uuids; |
79 | m_assetsRequestCallback = assetsRequestCallback; | 84 | m_assetsRequestCallback = assetsRequestCallback; |
80 | m_assetCache = assetCache; | 85 | m_assetCache = assetCache; |
@@ -87,7 +92,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
87 | 92 | ||
88 | // We can stop here if there are no assets to fetch | 93 | // We can stop here if there are no assets to fetch |
89 | if (m_repliesRequired == 0) | 94 | if (m_repliesRequired == 0) |
90 | m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); | 95 | m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids); |
91 | 96 | ||
92 | foreach (UUID uuid in m_uuids) | 97 | foreach (UUID uuid in m_uuids) |
93 | { | 98 | { |
@@ -106,21 +111,25 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
106 | 111 | ||
107 | if (asset != null) | 112 | if (asset != null) |
108 | { | 113 | { |
114 | // Make sure that we don't run out of memory by hogging assets in the cache | ||
109 | m_assetCache.ExpireAsset(assetID); | 115 | m_assetCache.ExpireAsset(assetID); |
110 | m_assets[assetID] = asset; | 116 | |
117 | m_foundAssetUuids.Add(assetID); | ||
118 | m_assetsArchiver.WriteAsset(asset); | ||
111 | } | 119 | } |
112 | else | 120 | else |
113 | { | 121 | { |
114 | m_notFoundAssetUuids.Add(assetID); | 122 | m_notFoundAssetUuids.Add(assetID); |
115 | } | 123 | } |
116 | 124 | ||
117 | if (m_assets.Count + m_notFoundAssetUuids.Count == m_repliesRequired) | 125 | if (m_foundAssetUuids.Count + m_notFoundAssetUuids.Count == m_repliesRequired) |
118 | { | 126 | { |
119 | m_log.DebugFormat( | 127 | m_log.DebugFormat( |
120 | "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets", | 128 | "[ARCHIVER]: Successfully received {0} assets and notification of {1} missing assets", |
121 | m_assets.Count, m_notFoundAssetUuids.Count); | 129 | m_foundAssetUuids.Count, m_notFoundAssetUuids.Count); |
122 | 130 | ||
123 | // We want to stop using the asset cache thread asap as we now need to do the actual work of producing the archive | 131 | // We want to stop using the asset cache thread asap |
132 | // as we now need to do the work of producing the rest of the archive | ||
124 | Thread newThread = new Thread(PerformAssetsRequestCallback); | 133 | Thread newThread = new Thread(PerformAssetsRequestCallback); |
125 | newThread.Name = "OpenSimulator archiving thread post assets receipt"; | 134 | newThread.Name = "OpenSimulator archiving thread post assets receipt"; |
126 | newThread.Start(); | 135 | newThread.Start(); |
@@ -134,7 +143,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver | |||
134 | { | 143 | { |
135 | try | 144 | try |
136 | { | 145 | { |
137 | m_assetsRequestCallback(m_assets, m_notFoundAssetUuids); | 146 | m_assetsRequestCallback(m_foundAssetUuids, m_notFoundAssetUuids); |
138 | } | 147 | } |
139 | catch (Exception e) | 148 | catch (Exception e) |
140 | { | 149 | { |