* Fixed about 7 issues with restarting sims and resolved interRegion comms issues. This includes the issue that MW described this morning.

There's a lot of little nit picky changes that make a world of difference.

6 files changed, 206 insertions, 134 deletions

diff --git a/OpenSim/Framework/Communications/IInterRegionCommunications.cs b/OpenSim/Framework/Communications/IInterRegionCommunications.cs
index a492c51..e025223 100644
--- a/OpenSim/Framework/Communications/IInterRegionCommunications.cs
+++ b/OpenSim/Framework/Communications/IInterRegionCommunications.cs
@@ -35,7 +35,7 @@ namespace OpenSim.Framework.Communications
         string rdebugRegionName{ get; set; }
         bool InformRegionOfChildAgent(ulong regionHandle, AgentCircuitData agentData);
         bool InformRegionOfPrimCrossing(ulong regionHandle, LLUUID primID, string objData);
-        bool RegionUp(SearializableRegionInfo region);
+        bool RegionUp(SearializableRegionInfo region, ulong regionhandle);
 
         bool ExpectAvatarCrossing(ulong regionHandle, LLUUID agentID, LLVector3 position, bool isFlying);
         bool ExpectPrimCrossing(ulong regionHandle, LLUUID primID, LLVector3 position, bool isFlying);
diff --git a/OpenSim/Region/Communications/Local/LocalBackEndServices.cs b/OpenSim/Region/Communications/Local/LocalBackEndServices.cs
index 0f3ec73..17989d4 100644
--- a/OpenSim/Region/Communications/Local/LocalBackEndServices.cs
+++ b/OpenSim/Region/Communications/Local/LocalBackEndServices.cs
@@ -194,34 +194,35 @@ namespace OpenSim.Region.Communications.Local
             return mapBlocks;
         }
 
-        /// <summary> 
-        /// </summary>
-        /// <param name="regionHandle"></param>
-        /// <param name="agentData"></param>
-        /// <returns></returns>
-        /// 
-        public bool RegionUp(SearializableRegionInfo sregion)
+        
+        public bool RegionUp(SearializableRegionInfo sregion, ulong regionhandle)
         {
             RegionInfo region = new RegionInfo(sregion);
-            foreach (RegionCommsListener listener in m_regionListeners.Values)
+            if (m_regionListeners.ContainsKey(regionhandle))
             {
-                listener.TriggerRegionUp(region);
+                return m_regionListeners[regionhandle].TriggerRegionUp(region);
             }
            
             return false;
         }
 
-        public bool TriggerRegionUp(RegionInfo region)
+        public bool TriggerRegionUp(RegionInfo region, ulong regionhandle)
         {
-            
-            foreach (RegionCommsListener listener in m_regionListeners.Values)
+
+            if (m_regionListeners.ContainsKey(regionhandle))
             {
-                listener.TriggerRegionUp(region);
+                return m_regionListeners[regionhandle].TriggerRegionUp(region);
             }
 
-            return true;
+            return false;
         }
 
+        /// <summary> 
+        /// </summary>
+        /// <param name="regionHandle"></param>
+        /// <param name="agentData"></param>
+        /// <returns></returns>
+        /// 
         public bool InformRegionOfChildAgent(ulong regionHandle, AgentCircuitData agentData)
             //should change from agentCircuitData
         {
diff --git a/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs b/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs
index b0442d2..b91f4c2 100644
--- a/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs
+++ b/OpenSim/Region/Communications/OGS1/OGS1GridServices.cs
@@ -248,39 +248,47 @@ namespace OpenSim.Region.Communications.OGS1
             }
             else
             {
-                Hashtable requestData = new Hashtable();
-                requestData["region_handle"] = regionHandle.ToString();
-                requestData["authkey"] = serversInfo.GridSendKey;
-                ArrayList SendParams = new ArrayList();
-                SendParams.Add(requestData);
-                XmlRpcRequest GridReq = new XmlRpcRequest("simulator_data_request", SendParams);
-                XmlRpcResponse GridResp = GridReq.Send(serversInfo.GridURL, 3000);
+                try
+                {
+                    Hashtable requestData = new Hashtable();
+                    requestData["region_handle"] = regionHandle.ToString();
+                    requestData["authkey"] = serversInfo.GridSendKey;
+                    ArrayList SendParams = new ArrayList();
+                    SendParams.Add(requestData);
+                    XmlRpcRequest GridReq = new XmlRpcRequest("simulator_data_request", SendParams);
+                    XmlRpcResponse GridResp = GridReq.Send(serversInfo.GridURL, 3000);
 
-                Hashtable responseData = (Hashtable) GridResp.Value;
+                    Hashtable responseData = (Hashtable)GridResp.Value;
 
-                if (responseData.ContainsKey("error"))
-                {
-                    Console.WriteLine("error received from grid server" + responseData["error"]);
-                    return null;
-                }
+                    if (responseData.ContainsKey("error"))
+                    {
+                        Console.WriteLine("error received from grid server" + responseData["error"]);
+                        return null;
+                    }
 
-                uint regX = Convert.ToUInt32((string) responseData["region_locx"]);
-                uint regY = Convert.ToUInt32((string) responseData["region_locy"]);
-                string internalIpStr = (string) responseData["sim_ip"];
-                uint port = Convert.ToUInt32(responseData["sim_port"]);
-                string externalUri = (string) responseData["sim_uri"];
+                    uint regX = Convert.ToUInt32((string)responseData["region_locx"]);
+                    uint regY = Convert.ToUInt32((string)responseData["region_locy"]);
+                    string internalIpStr = (string)responseData["sim_ip"];
+                    uint port = Convert.ToUInt32(responseData["sim_port"]);
+                    string externalUri = (string)responseData["sim_uri"];
 
-                IPEndPoint neighbourInternalEndPoint = new IPEndPoint(IPAddress.Parse(internalIpStr), (int) port);
-                string neighbourExternalUri = externalUri;
-                regionInfo = new RegionInfo(regX, regY, neighbourInternalEndPoint, internalIpStr);
+                    IPEndPoint neighbourInternalEndPoint = new IPEndPoint(IPAddress.Parse(internalIpStr), (int)port);
+                    string neighbourExternalUri = externalUri;
+                    regionInfo = new RegionInfo(regX, regY, neighbourInternalEndPoint, internalIpStr);
 
-                regionInfo.RemotingPort = Convert.ToUInt32((string) responseData["remoting_port"]);
-                regionInfo.RemotingAddress = internalIpStr;
+                    regionInfo.RemotingPort = Convert.ToUInt32((string)responseData["remoting_port"]);
+                    regionInfo.RemotingAddress = internalIpStr;
 
-                regionInfo.RegionID = new LLUUID((string) responseData["region_UUID"]);
-                regionInfo.RegionName = (string) responseData["region_name"];
+                    regionInfo.RegionID = new LLUUID((string)responseData["region_UUID"]);
+                    regionInfo.RegionName = (string)responseData["region_name"];
 
-                m_remoteRegionInfoCache.Add(regionHandle, regionInfo);
+                    m_remoteRegionInfoCache.Add(regionHandle, regionInfo);
+                }
+                catch (System.Net.WebException)
+                {
+                    MainLog.Instance.Error("GRID", "Region lookup failed for: " + regionHandle.ToString() + " - Is the GridServer down?");
+                    return null;
+                }
             }
 
             return regionInfo;
@@ -540,34 +548,33 @@ namespace OpenSim.Region.Communications.OGS1
         }
 
         // UGLY!
-        public bool RegionUp(SearializableRegionInfo region)
+        public bool RegionUp(SearializableRegionInfo region, ulong regionhandle)
         {
-            
-            // This is stupid.  For this to work, when the region registers it must request nearby map blocks.
-            // In addition to filling the map blocks, it fills a 'known regions' list.
-
-            // This known regions list then gets queried on here to get the remoting data from the neighbors.
-            // *Pull yourself up by your bootstraps?*
-            
-            if (m_localBackend.RegionUp(region))
+            SearializableRegionInfo regInfo = null;
+            try
             {
-                return true;
-            }
-            
-            foreach (SimpleRegionInfo knownregion in m_knownRegions)
-             {
-                 
-                 SearializableRegionInfo regInfo = new SearializableRegionInfo(RequestNeighbourInfo(knownregion.RegionID));
-
-                  try
-                  {
-                 
-                    if ((!(regInfo.Equals(null)) && regInfo.RemotingAddress.Length > 0))
+                // You may ask why this is in here...   
+                // The region asking the grid services about itself..  
+                // And, surprisingly, the reason is..  it doesn't know 
+                // it's own remoting port!  How special.
+                region = new SearializableRegionInfo(RequestNeighbourInfo(region.RegionHandle));
+                region.RemotingAddress = region.ExternalHostName;
+                region.RemotingPort = (uint) NetworkServersInfo.RemotingListenerPort;
+                if (m_localBackend.RegionUp(region,regionhandle))
+                {
+                    return true;
+                }
+
+                regInfo = new SearializableRegionInfo(RequestNeighbourInfo(regionhandle));
+                if (regInfo != null)
+                {
+                    // If we're not trying to remote to ourselves.
+                    if (regInfo.RemotingAddress != region.RemotingAddress && region.RemotingAddress != null)
                     {
                         //don't want to be creating a new link to the remote instance every time like we are here
                         bool retValue = false;
 
-                        
+
                         OGS1InterRegionRemoting remObject = (OGS1InterRegionRemoting)Activator.GetObject(
                                                                                           typeof(OGS1InterRegionRemoting),
                                                                                           "tcp://" + regInfo.RemotingAddress +
@@ -576,49 +583,59 @@ namespace OpenSim.Region.Communications.OGS1
 
                         if (remObject != null)
                         {
-                            retValue = remObject.RegionUp(region);
+                            retValue = remObject.RegionUp(region, regionhandle);
                         }
                         else
                         {
                             Console.WriteLine("remoting object not found");
                         }
                         remObject = null;
-                        MainLog.Instance.Verbose("INTER", gdebugRegionName + ": OGS1 tried to NotifyRegionUp for " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                        MainLog.Instance.Verbose("INTER", gdebugRegionName + ": OGS1 tried to inform region I'm up");
 
+                        return retValue;
+                    }
+                    else
+                    {
+                        // We're trying to inform ourselves via remoting. 
+                        // This is here because we're looping over the listeners before we get here.
+                        // Odd but it should work.
+                        return true;
                     }
-
-                }
-                catch (RemotingException e)
-                {
-                    MainLog.Instance.Warn("Remoting Error: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
-                    MainLog.Instance.Debug(e.ToString());
-                    //return false;
-                }
-                catch (SocketException e)
-                {
-                    MainLog.Instance.Warn("Socket Error: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
-                    MainLog.Instance.Debug(e.ToString());
-                    //return false;
-                }
-                catch (InvalidCredentialException e)
-                {
-                    MainLog.Instance.Warn("Invalid Credentials: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
-                    MainLog.Instance.Debug(e.ToString());
-                    //return false;
-                }
-                catch (AuthenticationException e)
-                {
-                    MainLog.Instance.Warn("Authentication exception: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
-                    MainLog.Instance.Debug(e.ToString());
-                    //return false;
-                }
-                catch (Exception e)
-                {
-                    MainLog.Instance.Warn("Unknown exception: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
-                    MainLog.Instance.Debug(e.ToString());
-                    //return false;
                 }
+
+                return false;
+            }
+            catch (RemotingException e)
+            {
+                MainLog.Instance.Warn("Remoting Error: Unable to connect to adjacent region: " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                MainLog.Instance.Debug(e.ToString());
+                return false;
+            }
+            catch (SocketException e)
+            {
+                MainLog.Instance.Warn("Socket Error: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                MainLog.Instance.Debug(e.ToString());
+                return false;
             }
+            catch (InvalidCredentialException e)
+            {
+                MainLog.Instance.Warn("Invalid Credentials: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                MainLog.Instance.Debug(e.ToString());
+                return false;
+            }
+            catch (AuthenticationException e)
+            {
+                MainLog.Instance.Warn("Authentication exception: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                MainLog.Instance.Debug(e.ToString());
+                return false;
+            }
+            catch (Exception e)
+            {
+                MainLog.Instance.Warn("Unknown exception: Unable to connect to adjacent region:  " + regInfo.RegionLocX + "," + regInfo.RegionLocY);
+                MainLog.Instance.Debug(e.ToString());
+                return false;
+            }
+            
             return true;
         }
         /// <summary>
@@ -837,13 +854,13 @@ namespace OpenSim.Region.Communications.OGS1
             }
         }
 
-        public bool TriggerRegionUp(SearializableRegionInfo regionData)
+        public bool TriggerRegionUp(SearializableRegionInfo regionData, ulong regionhandle)
         {
             MainLog.Instance.Verbose("INTER", gdebugRegionName + ": Incoming OGS1 RegionUpReport:  " + regionData.RegionLocX + "," + regionData.RegionLocY);
 
             try
             {
-                return m_localBackend.TriggerRegionUp(new RegionInfo(regionData));
+                return m_localBackend.TriggerRegionUp(new RegionInfo(regionData), regionhandle);
             }
 
             catch (RemotingException e)
diff --git a/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs b/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs
index 2f7e591..d55528c 100644
--- a/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs
+++ b/OpenSim/Region/Communications/OGS1/OGS1InterSimComms.cs
@@ -41,7 +41,7 @@ namespace OpenSim.Region.Communications.OGS1
 
     public delegate bool PrimGroupArrival(ulong regionHandle, LLUUID primID, string objData);
 
-    public delegate bool RegionUp (SearializableRegionInfo region);
+    public delegate bool RegionUp (SearializableRegionInfo region, ulong regionhandle);
 
     public sealed class InterRegionSingleton
     {
@@ -76,12 +76,12 @@ namespace OpenSim.Region.Communications.OGS1
             return false;
         }
 
-        public bool RegionUp(SearializableRegionInfo sregion)
+        public bool RegionUp(SearializableRegionInfo sregion, ulong regionhandle)
         {
             
             if (OnRegionUp != null)
             {
-                return OnRegionUp(sregion);
+                return OnRegionUp(sregion, regionhandle);
             }
             return false;
         }
@@ -130,11 +130,11 @@ namespace OpenSim.Region.Communications.OGS1
                 return false;
             }
         }
-        public bool RegionUp(SearializableRegionInfo  region)
+        public bool RegionUp(SearializableRegionInfo  region, ulong regionhandle)
         {
             try
             {
-                return InterRegionSingleton.Instance.RegionUp(region);
+                return InterRegionSingleton.Instance.RegionUp(region, regionhandle);
             }
             catch (RemotingException e)
             {
diff --git a/OpenSim/Region/Environment/Scenes/Scene.cs b/OpenSim/Region/Environment/Scenes/Scene.cs
index 68af360..3309431 100644
--- a/OpenSim/Region/Environment/Scenes/Scene.cs
+++ b/OpenSim/Region/Environment/Scenes/Scene.cs
@@ -277,15 +277,41 @@ namespace OpenSim.Region.Environment.Scenes
             // This fails to get the desired effect and needs further work.
             if (RegionInfo.RegionHandle != otherRegion.RegionHandle)
             {
-                if (!(m_regionRestartNotifyList.Contains(otherRegion)))
+                if (Math.Abs(otherRegion.RegionLocX - RegionInfo.RegionLocX) <= 1 || Math.Abs(otherRegion.RegionLocY - RegionInfo.RegionLocY) <= 1)
                 {
-                    m_regionRestartNotifyList.Add(otherRegion);
-                    
-                    m_restartWaitTimer = new Timer(20000);
-                    m_restartWaitTimer.AutoReset = false;
-                    m_restartWaitTimer.Elapsed += new ElapsedEventHandler(restart_Notify_Wait_Elapsed);
-                    m_restartWaitTimer.Start();
+                    try
+                    {
+
+                        ForEachScenePresence(delegate(ScenePresence agent)
+                        {
+                            if (!(agent.IsChildAgent))
+                            {
+                                //agent.ControllingClient.new
+                                //this.CommsManager.InterRegion.InformRegionOfChildAgent(otherRegion.RegionHandle, agent.ControllingClient.RequestClientInfo());
+                                InformClientOfNeighbor(agent, otherRegion);
+                            }
+                        }
+
+                        );
+                    }
+                    catch (System.NullReferenceException)
+                    {
+                        // This means that we're not booted up completely yet.
+                    }
+                }
+                else
+                {
+                    MainLog.Instance.Verbose("INTERGRID", "Got notice about Region at X:" + otherRegion.RegionLocX.ToString() + " Y:" + otherRegion.RegionLocY.ToString() + " but it was too far away to send to the client");
                 }
+                //if (!(m_regionRestartNotifyList.Contains(otherRegion)))
+                //{
+                    //m_regionRestartNotifyList.Add(otherRegion);
+                    
+                    //m_restartWaitTimer = new Timer(20000);
+                    //m_restartWaitTimer.AutoReset = false;
+                   // m_restartWaitTimer.Elapsed += new ElapsedEventHandler(restart_Notify_Wait_Elapsed);
+                    //m_restartWaitTimer.Start();
+                //}
             }
             return true;
         }
@@ -342,25 +368,7 @@ namespace OpenSim.Region.Environment.Scenes
             m_restartWaitTimer.Stop();
             foreach (RegionInfo region in m_regionRestartNotifyList)
             {
-                try
-                {
-
-                    ForEachScenePresence(delegate(ScenePresence agent)
-                    {
-                        if (!(agent.IsChildAgent))
-                        {
-                            //agent.ControllingClient.new
-                            //this.CommsManager.InterRegion.InformRegionOfChildAgent(otherRegion.RegionHandle, agent.ControllingClient.RequestClientInfo());
-                            InformClientOfNeighbor(agent, region);
-                        }
-                    }
-
-                    );
-                }
-                catch (System.NullReferenceException)
-                {
-                    // This means that we're not booted up completely yet.
-                }
+                
             }
             // Reset list to nothing.
             m_regionRestartNotifyList = new List<RegionInfo>();
diff --git a/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs b/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs
index 892bf81..aad78d5 100644
--- a/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs
+++ b/OpenSim/Region/Environment/Scenes/SceneCommunicationService.cs
@@ -58,6 +58,7 @@ namespace OpenSim.Region.Environment.Scenes
                 //MainLog.Instance.Verbose("INTER", debugRegionName + ": SceneCommunicationService: registered with gridservice and got" + regionCommsHost.ToString());
 
                 regionCommsHost.debugRegionName = _debugRegionName;
+
                 regionCommsHost.OnExpectUser += NewUserConnection;
                 regionCommsHost.OnAvatarCrossingIntoRegion += AgentCrossing;
                 regionCommsHost.OnPrimCrossingIntoRegion += PrimCrossing;
@@ -222,7 +223,56 @@ namespace OpenSim.Region.Environment.Scenes
         }
 
         #endregion
+        public delegate void InformNeighbourThatRegionUpDelegate(RegionInfo region, ulong regionhandle);
+
+        private void InformNeighborsThatRegionisUpCompleted(IAsyncResult iar)
+        {
+            InformNeighbourThatRegionUpDelegate icon = (InformNeighbourThatRegionUpDelegate)iar.AsyncState;
+            icon.EndInvoke(iar);
+        }
+
+
+         private void InformNeighboursThatRegionIsUpAsync(RegionInfo region, ulong regionhandle)
+        {
+            MainLog.Instance.Notice("INTERGRID", "Starting to inform neighbors that I'm here");
+            bool regionAccepted = m_commsProvider.InterRegion.RegionUp((new SearializableRegionInfo(region)), regionhandle);
+
+            if (regionAccepted)
+            {
+                MainLog.Instance.Notice("INTERGRID", "Completed informing neighbors that I'm here");
+            }
+            else
+            {
+                MainLog.Instance.Notice("INTERGRID", "Failed to inform neighbors that I'm here");
+            }
+        }
 
+        public void InformNeighborsThatRegionisUp(RegionInfo region)
+        {
+            //MainLog.Instance.Verbose("INTER", debugRegionName + ": SceneCommunicationService: Sending InterRegion Notification that region is up " + region.RegionName);
+            List<SimpleRegionInfo> neighbours = new List<SimpleRegionInfo>();
+                
+             lock (neighbours)
+             {
+                 neighbours = m_commsProvider.GridService.RequestNeighbours(m_regionInfo.RegionLocX, m_regionInfo.RegionLocY);
+                 if (neighbours != null)
+                 {
+                     for (int i = 0; i < neighbours.Count; i++)
+                     {
+
+                         InformNeighbourThatRegionUpDelegate d = InformNeighboursThatRegionIsUpAsync;
+                         // race condition!  Arg!  I hate race conditions.
+                         lock (d)
+                         {
+                             d.BeginInvoke(region, neighbours[i].RegionHandle,
+                                           InformNeighborsThatRegionisUpCompleted,
+                                           d);
+                         }
+                     }
+                 }
+             }
+            //bool val = m_commsProvider.InterRegion.RegionUp(new SearializableRegionInfo(region));
+        }
         /// <summary>
         /// 
         /// </summary>
@@ -320,11 +370,7 @@ namespace OpenSim.Region.Environment.Scenes
             return m_commsProvider.InterRegion.ExpectAvatarCrossing(regionhandle, agentID, position, isFlying);
         }
 
-        public void InformNeighborsThatRegionisUp(RegionInfo region)
-        {
-            //MainLog.Instance.Verbose("INTER", debugRegionName + ": SceneCommunicationService: Sending InterRegion Notification that region is up " + region.RegionName);
-            bool val = m_commsProvider.InterRegion.RegionUp(new SearializableRegionInfo(region));
-        }
+
 
         public bool PrimCrossToNeighboringRegion(ulong regionhandle, LLUUID primID, LLVector3 position, bool isPhysical)
         {