4 files changed, 121 insertions, 29 deletions

diff --git a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs
index be409bb..7e356ea 100644
--- a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs
+++ b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs
@@ -546,6 +546,31 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
             }
         }
 
+        private void RemoveIncomingSceneObjectJobs(string commonIdToRemove)
+        {
+            List<Job> jobsToReinsert = new List<Job>();
+            int jobsRemoved = 0;
+
+            Job job;
+            while ((job = m_incomingSceneObjectEngine.RemoveNextRequest()) != null)
+            {
+                if (job.CommonId != commonIdToRemove)
+                    jobsToReinsert.Add(job);
+                else
+                    jobsRemoved++;
+            }
+
+            m_log.DebugFormat(
+                "[HG ENTITY TRANSFER]: Removing {0} jobs with common ID {1} and reinserting {2} other jobs",
+                jobsRemoved, commonIdToRemove, jobsToReinsert.Count);
+
+            if (jobsToReinsert.Count > 0)
+            {                                        
+                foreach (Job jobToReinsert in jobsToReinsert)
+                    m_incomingSceneObjectEngine.QueueRequest(jobToReinsert);
+            }
+        }
+
         public override bool HandleIncomingSceneObject(SceneObjectGroup so, Vector3 newPosition)
         {
             // FIXME: We must make it so that we can use SOG.IsAttachment here.  At the moment it is always null!
@@ -564,38 +589,73 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
                 {
                     m_incomingSceneObjectEngine.QueueRequest(
                         string.Format("HG UUID Gather for attachment {0} for {1}", so.Name, aCircuit.Name), 
+                        so.OwnerID.ToString(),
                         o => 
                         {
                             string url = aCircuit.ServiceURLs["AssetServerURI"].ToString();
-                            m_log.DebugFormat(
-                                "[HG ENTITY TRANSFER MODULE]: Incoming attachment {0} for HG user {1} with asset server {2}", 
-                                so.Name, so.AttachedAvatar, url);
+//                            m_log.DebugFormat(
+//                                "[HG ENTITY TRANSFER MODULE]: Incoming attachment {0} for HG user {1} with asset service {2}", 
+//                                so.Name, so.AttachedAvatar, url);
 
                             IteratingHGUuidGatherer uuidGatherer = new IteratingHGUuidGatherer(Scene.AssetService, url);
                             uuidGatherer.RecordAssetUuids(so);
-                            
-                            // XXX: We will shortly use this iterating mechanism to check if a fetch is taking too long
-                            // but just for now we will simply fetch everything.  If this was permanent could use
-                            // GatherAll()
-                            while (uuidGatherer.GatherNext())
-                                m_log.DebugFormat(
-                                    "[HG ENTITY TRANSFER]: Gathered attachment {0} for HG user {1} with asset server {2}",
-                                    so.Name, so.OwnerID, url);
+
+                            while (!uuidGatherer.Complete)
+                            {
+                                int tickStart = Util.EnvironmentTickCount();
+
+                                UUID? nextUuid = uuidGatherer.NextUuidToInspect;
+                                uuidGatherer.GatherNext();
+
+//                                m_log.DebugFormat(
+//                                    "[HG ENTITY TRANSFER]: Gathered attachment asset uuid {0} for object {1} for HG user {2} took {3} ms with asset service {4}",
+//                                    nextUuid, so.Name, so.OwnerID, Util.EnvironmentTickCountSubtract(tickStart), url);
+
+                                int ticksElapsed = Util.EnvironmentTickCountSubtract(tickStart);
+
+                                if (ticksElapsed > 30000)
+                                {
+                                    m_log.WarnFormat(
+                                        "[HG ENTITY TRANSFER]: Removing incoming scene object jobs for HG user {0} as gather of {1} from {2} took {3} ms to respond (> {4} ms)",
+                                        so.OwnerID, so.Name, url, ticksElapsed, 30000);
+
+                                    RemoveIncomingSceneObjectJobs(so.OwnerID.ToString());
+
+                                    return;
+                                }                                                           
+                            }
 
                             IDictionary<UUID, sbyte> ids = uuidGatherer.GetGatheredUuids();
 
-                            m_log.DebugFormat(
-                                "[HG ENTITY TRANSFER]: Fetching {0} assets for attachment {1} for HG user {2} with asset server {3}",
-                                ids.Count, so.Name, so.OwnerID, url);
+//                            m_log.DebugFormat(
+//                                "[HG ENTITY TRANSFER]: Fetching {0} assets for attachment {1} for HG user {2} with asset service {3}",
+//                                ids.Count, so.Name, so.OwnerID, url);
 
                             foreach (KeyValuePair<UUID, sbyte> kvp in ids)
-                                uuidGatherer.FetchAsset(kvp.Key);
+                            {
+                                int tickStart = Util.EnvironmentTickCount();
+
+                                uuidGatherer.FetchAsset(kvp.Key);   
+
+                                int ticksElapsed = Util.EnvironmentTickCountSubtract(tickStart);
+
+                                if (ticksElapsed > 30000)
+                                {
+                                    m_log.WarnFormat(
+                                        "[HG ENTITY TRANSFER]: Removing incoming scene object jobs for HG user {0} as fetch of {1} from {2} took {3} ms to respond (> {4} ms)",
+                                        so.OwnerID, kvp.Key, url, ticksElapsed, 30000);
+
+                                    RemoveIncomingSceneObjectJobs(so.OwnerID.ToString());
+
+                                    return;
+                                }   
+                            }
 
                             base.HandleIncomingSceneObject(so, newPosition);
 
-                            m_log.DebugFormat(
-                                "[HG ENTITY TRANSFER MODULE]: Completed incoming attachment {0} for HG user {1} with asset server {2}", 
-                                so.Name, so.OwnerID, url);
+//                            m_log.DebugFormat(
+//                                "[HG ENTITY TRANSFER MODULE]: Completed incoming attachment {0} for HG user {1} with asset server {2}", 
+//                                so.Name, so.OwnerID, url);
                         }, 
                         null);
                 }
diff --git a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs
index 718db31..ce6cdc9 100644
--- a/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs
+++ b/OpenSim/Region/CoreModules/Framework/EntityTransfer/HGIncomingSceneObjectEngine.cs
@@ -38,13 +38,15 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
 {
     public class Job
     {
-        public string Name;
-        public WaitCallback Callback;
-        public object O;
+        public string Name { get; private set; }
+        public string CommonId { get; private set; }
+        public WaitCallback Callback { get; private set; }
+        public object O { get; private set; }
 
-        public Job(string name, WaitCallback callback, object o)
+        public Job(string name, string commonId, WaitCallback callback, object o)
         {
             Name = name;
+            CommonId = commonId;
             Callback = callback;
             O = o;
         }
@@ -90,6 +92,7 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
 
         public HGIncomingSceneObjectEngine(string name)
         {
+//            LogLevel = 1;
             Name = name;
             RequestProcessTimeoutOnStop = 5000;
 
@@ -192,10 +195,24 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
             }
         }
 
-        public bool QueueRequest(string name, WaitCallback req, object o)
+        public Job RemoveNextRequest()
+        {
+            Job nextRequest;
+            m_requestQueue.TryTake(out nextRequest);
+
+            return nextRequest;
+        }
+
+        public bool QueueRequest(string name, string commonId, WaitCallback req, object o)
+        {
+            return QueueRequest(new Job(name, commonId, req, o));
+        }
+
+        public bool QueueRequest(Job job)
         {
             if (LogLevel >= 1)
-                m_log.DebugFormat("[HG INCOMING SCENE OBJECT ENGINE]: Queued job {0}", name);
+                m_log.DebugFormat(
+                    "[HG INCOMING SCENE OBJECT ENGINE]: Queued job {0}, common ID {1}", job.Name, job.CommonId);
 
             if (m_requestQueue.Count < m_requestQueue.BoundedCapacity)
             {
@@ -203,7 +220,7 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
                 //                    "[OUTGOING QUEUE REFILL ENGINE]: Adding request for categories {0} for {1} in {2}", 
                 //                    categories, client.AgentID, m_udpServer.Scene.Name);
 
-                m_requestQueue.Add(new Job(name, req, o));
+                m_requestQueue.Add(job);
 
                 if (!m_warnOverMaxQueue)
                     m_warnOverMaxQueue = true;
diff --git a/OpenSim/Region/Framework/Scenes/UuidGatherer.cs b/OpenSim/Region/Framework/Scenes/UuidGatherer.cs
index 9c4e4c0..2450cdb 100644
--- a/OpenSim/Region/Framework/Scenes/UuidGatherer.cs
+++ b/OpenSim/Region/Framework/Scenes/UuidGatherer.cs
@@ -655,7 +655,22 @@ namespace OpenSim.Region.Framework.Scenes
         /// <summary>
         /// Is gathering complete?
         /// </summary>
-        public bool GatheringComplete { get { return m_assetUuidsToInspect.Count <= 0; } }
+        public bool Complete { get { return m_assetUuidsToInspect.Count <= 0; } }
+
+        /// <summary>
+        /// Gets the next UUID to inspect.
+        /// </summary>
+        /// <value>If there is no next UUID then returns null</value>
+        public UUID? NextUuidToInspect
+        {
+            get
+            {
+                if (Complete)
+                    return null;
+                else
+                    return m_assetUuidsToInspect.Peek();
+            }
+        }
 
         protected IAssetService m_assetService;
 
@@ -693,7 +708,7 @@ namespace OpenSim.Region.Framework.Scenes
         /// <returns>false if gathering is already complete, true otherwise</returns>
         public bool GatherNext()
         {
-            if (GatheringComplete)
+            if (Complete)
                 return false;
 
             GetAssetUuids(m_assetUuidsToInspect.Dequeue());
@@ -707,7 +722,7 @@ namespace OpenSim.Region.Framework.Scenes
         /// <returns>false if gathering is already complete, true otherwise</returns>
         public bool GatherAll()
         {
-            if (GatheringComplete)
+            if (Complete)
                 return false;
 
             while (GatherNext());
diff --git a/OpenSim/Services/HypergridService/HGAssetService.cs b/OpenSim/Services/HypergridService/HGAssetService.cs
index 5c804d4..b57f8d8 100644
--- a/OpenSim/Services/HypergridService/HGAssetService.cs
+++ b/OpenSim/Services/HypergridService/HGAssetService.cs
@@ -100,7 +100,7 @@ namespace OpenSim.Services.HypergridService
                 return null;
 
             if (asset.Metadata.Type == (sbyte)AssetType.Object)
-                asset.Data = AdjustIdentifiers(asset.Data); ;
+                asset.Data = AdjustIdentifiers(asset.Data);
 
             AdjustIdentifiers(asset.Metadata);