From a5b9971fd77c0c4bf70656be7f3e7999f59d9f85 Mon Sep 17 00:00:00 2001
From: John Hurliman
Date: Fri, 9 Oct 2009 01:53:06 -0700
Subject: * Added a lock object for the write functions in
LLUDPClientCollection (immutable != concurrent write safety) * Allow the UDP
server to bind to a user-specified port again * Updated to a newer version of
OpenSimUDPBase that streamlines the code even more. This also reintroduces
the highly concurrent packet handling which needs more testing
---
.../ClientStack/LindenUDP/LLUDPClientCollection.cs | 56 +++-
.../Region/ClientStack/LindenUDP/LLUDPServer.cs | 20 +-
.../Region/ClientStack/LindenUDP/OpenSimUDPBase.cs | 303 ++++++++-------------
3 files changed, 169 insertions(+), 210 deletions(-)
(limited to 'OpenSim/Region')
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClientCollection.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClientCollection.cs
index abf3882..2222a33 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPClientCollection.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPClientCollection.cs
@@ -36,6 +36,9 @@ using ReaderWriterLockImpl = OpenMetaverse.ReaderWriterLockSlim;
namespace OpenSim.Region.ClientStack.LindenUDP
{
+ /// <summary>
+ /// A thread safe mapping from endpoints to client references
+ /// </summary>
public sealed class UDPClientCollection
{
#region IComparers
@@ -52,43 +55,80 @@ namespace OpenSim.Region.ClientStack.LindenUDP
#endregion IComparers
+ /// <summary>An immutable dictionary mapping from <seealso cref="IPEndPoint"/>
+ /// to <seealso cref="LLUDPClient"/> references</summary>
private ImmutableMap<IPEndPoint, LLUDPClient> m_dict;
+ /// <summary>Immutability grants thread safety for concurrent reads and
+ /// read-writes, but not concurrent writes</summary>
+ private object m_writeLock;
+ /// <summary>Number of clients in the collection</summary>
+ public int Count { get { return m_dict.Count; } }
+
+ /// <summary>
+ /// Default constructor
+ /// </summary>
public UDPClientCollection()
{
m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
}
+ /// <summary>
+ /// Add a client reference to the collection
+ /// </summary>
+ /// <param name="key">Remote endpoint of the client</param>
+ /// <param name="value">Reference to the client object</param>
public void Add(IPEndPoint key, LLUDPClient value)
{
- m_dict = m_dict.Add(key, value);
+ lock (m_writeLock)
+ m_dict = m_dict.Add(key, value);
}
+ /// <summary>
+ /// Remove a client from the collection
+ /// </summary>
+ /// <param name="key">Remote endpoint of the client</param>
public void Remove(IPEndPoint key)
{
- m_dict = m_dict.Delete(key);
+ lock (m_writeLock)
+ m_dict = m_dict.Delete(key);
}
+ /// <summary>
+ /// Resets the client collection
+ /// </summary>
public void Clear()
{
- m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
- }
-
- public int Count
- {
- get { return m_dict.Count; }
+ lock (m_writeLock)
+ m_dict = new ImmutableMap<IPEndPoint, LLUDPClient>(new IPEndPointComparer());
}
+ /// <summary>
+ /// Checks if an endpoint is in the collection
+ /// </summary>
+ /// <param name="key">Endpoint to check for</param>
+ /// <returns>True if the endpoint was found in the collection, otherwise false</returns>
public bool ContainsKey(IPEndPoint key)
{
return m_dict.ContainsKey(key);
}
+ /// <summary>
+ /// Attempts to fetch a value out of the collection
+ /// </summary>
+ /// <param name="key">Endpoint of the client to retrieve</param>
+ /// <param name="value">Retrieved client, or null on lookup failure</param>
+ /// <returns>True if the lookup succeeded, otherwise false</returns>
public bool TryGetValue(IPEndPoint key, out LLUDPClient value)
{
return m_dict.TryGetValue(key, out value);
}
+ /// <summary>
+ /// Performs a given task in parallel for each of the elements in the
+ /// collection
+ /// </summary>
+ /// <param name="action">Action to perform on each element</param>
public void ForEach(Action<LLUDPClient> action)
{
Parallel.ForEach<LLUDPClient>(m_dict.Values, action);
diff --git a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
index a6aa048..9aeea9a 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
@@ -96,7 +96,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
/// <summary>Incoming packets that are awaiting handling</summary>
private OpenMetaverse.BlockingQueue<IncomingPacket> packetInbox = new OpenMetaverse.BlockingQueue<IncomingPacket>();
/// <summary></summary>
- private UDPClientCollection clients = new UDPClientCollection();
+ private UDPClientCollection m_clients = new UDPClientCollection();
/// <summary>Bandwidth throttle for this UDP server</summary>
private TokenBucket m_throttle;
/// <summary>Bandwidth throttle rates for this UDP server</summary>
@@ -115,7 +115,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public Socket Server { get { return null; } }
public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
- : base((int)port)
+ : base(listenIP, (int)port)
{
#region Environment.TickCount Measurement
@@ -143,7 +143,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
public new void Start()
{
if (m_scene == null)
- throw new InvalidOperationException("Cannot LLUDPServer.Start() without an IScene reference");
+ throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
base.Start();
@@ -188,7 +188,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
m_scene.ClientManager.Remove(udpClient.CircuitCode);
udpClient.ClientAPI.Close(false);
udpClient.Shutdown();
- clients.Remove(udpClient.RemoteEndPoint);
+ m_clients.Remove(udpClient.RemoteEndPoint);
}
public void BroadcastPacket(Packet packet, ThrottleOutPacketType category, bool sendToPausedAgents, bool allowSplitting)
@@ -208,7 +208,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
for (int i = 0; i < packetCount; i++)
{
byte[] data = datas[i];
- clients.ForEach(
+ m_clients.ForEach(
delegate(LLUDPClient client)
{ SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); });
}
@@ -216,7 +216,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
else
{
byte[] data = packet.ToBytes();
- clients.ForEach(
+ m_clients.ForEach(
delegate(LLUDPClient client)
{ SendPacketData(client, data, data.Length, packet.Type, packet.Header.Zerocoded, category); });
}
@@ -502,7 +502,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
// Determine which agent this packet came from
- if (!clients.TryGetValue(address, out client))
+ if (!m_clients.TryGetValue(address, out client))
{
m_log.Warn("[LLUDPSERVER]: Received a " + packet.Type + " packet from an unrecognized source: " + address);
return;
@@ -606,7 +606,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (m_scene.RegionStatus != RegionStatus.SlaveScene)
{
AuthenticateResponse sessionInfo;
- bool isNewCircuit = !clients.ContainsKey(remoteEndPoint);
+ bool isNewCircuit = !m_clients.ContainsKey(remoteEndPoint);
if (!IsClientAuthorized(useCircuitCode, out sessionInfo))
{
@@ -648,7 +648,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
udpClient.ClientAPI = clientApi;
// Add the new client to our list of tracked clients
- clients.Add(udpClient.RemoteEndPoint, udpClient);
+ m_clients.Add(udpClient.RemoteEndPoint, udpClient);
}
private void AcknowledgePacket(LLUDPClient client, uint ack, int currentTime, bool fromResend)
@@ -726,7 +726,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
elapsed500MS = 0;
}
- clients.ForEach(
+ m_clients.ForEach(
delegate(LLUDPClient client)
{
if (client.DequeueOutgoing())
diff --git a/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs b/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
index 218aaac..9b1751d 100644
--- a/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
+++ b/OpenSim/Region/ClientStack/LindenUDP/OpenSimUDPBase.cs
@@ -29,101 +29,90 @@ using System;
using System.Net;
using System.Net.Sockets;
using System.Threading;
-using OpenMetaverse;
+using log4net;
-namespace OpenSim.Region.ClientStack.LindenUDP
+namespace OpenMetaverse
{
/// <summary>
- ///
+ /// Base UDP server
/// </summary>
public abstract class OpenSimUDPBase
{
- // these abstract methods must be implemented in a derived class to actually do
- // something with the packets that are sent and received.
+ private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
+
+ /// <summary>
+ /// This method is called when an incoming packet is received
+ /// </summary>
+ /// <param name="buffer">Incoming packet buffer</param>
protected abstract void PacketReceived(UDPPacketBuffer buffer);
+
+ /// <summary>
+ /// This method is called when an outgoing packet is sent
+ /// </summary>
+ /// <param name="buffer">Outgoing packet buffer</param>
+ /// <param name="bytesSent">Number of bytes written to the wire</param>
protected abstract void PacketSent(UDPPacketBuffer buffer, int bytesSent);
- // the port to listen on
- internal int udpPort;
-
- // the UDP socket
- private Socket udpSocket;
+ /// <summary>UDP port to bind to in server mode</summary>
+ protected int m_udpPort;
- // the ReaderWriterLock is used solely for the purposes of shutdown (Stop()).
- // since there are potentially many "reader" threads in the internal .NET IOCP
- // thread pool, this is a cheaper synchronization primitive than using
- // a Mutex object. This allows many UDP socket "reads" concurrently - when
- // Stop() is called, it attempts to obtain a writer lock which will then
- // wait until all outstanding operations are completed before shutting down.
- // this avoids the problem of closing the socket with outstanding operations
- // and trying to catch the inevitable ObjectDisposedException.
- private ReaderWriterLock rwLock = new ReaderWriterLock();
+ /// <summary>Local IP address to bind to in server mode</summary>
+ protected IPAddress m_localBindAddress;
- // number of outstanding operations. This is a reference count
- // which we use to ensure that the threads exit cleanly. Note that
- // we need this because the threads will potentially still need to process
- // data even after the socket is closed.
- private int rwOperationCount = 0;
+ /// <summary>UDP socket, used in either client or server mode</summary>
+ private Socket m_udpSocket;
- // the all important shutdownFlag. This is synchronized through the ReaderWriterLock.
- private volatile bool shutdownFlag = true;
-
- // the remote endpoint to communicate with
- protected IPEndPoint remoteEndPoint = null;
+ /// <summary>The all important shutdown flag</summary>
+ private volatile bool m_shutdownFlag = true;
+ /// <summary>Returns true if the server is currently listening, otherwise false</summary>
+ public bool IsRunning { get { return !m_shutdownFlag; } }
/// <summary>
- /// Initialize the UDP packet handler in server mode
+ /// Default constructor
/// </summary>
+ /// <param name="bindAddress">Local IP address to bind the server to</param>
/// <param name="port">Port to listening for incoming UDP packets on</param>
- public OpenSimUDPBase(int port)
- {
- udpPort = port;
- }
-
- /// <summary>
- /// Initialize the UDP packet handler in client mode
- /// </summary>
- /// <param name="endPoint">Remote UDP server to connect to</param>
- public OpenSimUDPBase(IPEndPoint endPoint)
+ public OpenSimUDPBase(IPAddress bindAddress, int port)
{
- remoteEndPoint = endPoint;
- udpPort = 0;
+ m_localBindAddress = bindAddress;
+ m_udpPort = port;
}
/// <summary>
- ///
+ /// Start the UDP server
/// </summary>
+ /// <remarks>This method will attempt to set the SIO_UDP_CONNRESET flag
+ /// on the socket to get newer versions of Windows to behave in a sane
+ /// manner (not throwing an exception when the remote side resets the
+ /// connection). This call is ignored on Mono where the flag is not
+ /// necessary</remarks>
public void Start()
{
- if (shutdownFlag)
+ if (m_shutdownFlag)
{
- if (remoteEndPoint == null)
- {
- // Server mode
+ const int SIO_UDP_CONNRESET = -1744830452;
- // create and bind the socket
- IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
- udpSocket = new Socket(
- AddressFamily.InterNetwork,
- SocketType.Dgram,
- ProtocolType.Udp);
- udpSocket.Bind(ipep);
+ IPEndPoint ipep = new IPEndPoint(m_localBindAddress, m_udpPort);
+ m_udpSocket = new Socket(
+ AddressFamily.InterNetwork,
+ SocketType.Dgram,
+ ProtocolType.Udp);
+ try
+ {
+ // this udp socket flag is not supported under mono,
+ // so we'll catch the exception and continue
+ m_udpSocket.IOControl(SIO_UDP_CONNRESET, new byte[] { 0 }, null);
+ m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag set");
}
- else
+ catch (SocketException)
{
- // Client mode
- IPEndPoint ipep = new IPEndPoint(Settings.BIND_ADDR, udpPort);
- udpSocket = new Socket(
- AddressFamily.InterNetwork,
- SocketType.Dgram,
- ProtocolType.Udp);
- udpSocket.Bind(ipep);
- //udpSocket.Connect(remoteEndPoint);
+ m_log.Debug("[UDPBASE]: SIO_UDP_CONNRESET flag not supported on this platform, ignoring");
}
+ m_udpSocket.Bind(ipep);
// we're not shutting down, we're starting up
- shutdownFlag = false;
+ m_shutdownFlag = false;
// kick off an async receive. The Start() method will return, the
// actual receives will occur asynchronously and will be caught in
@@ -133,104 +122,85 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
/// <summary>
- ///
+ /// Stops the UDP server
/// </summary>
public void Stop()
{
- if (!shutdownFlag)
+ if (!m_shutdownFlag)
{
// wait indefinitely for a writer lock. Once this is called, the .NET runtime
// will deny any more reader locks, in effect blocking all other send/receive
// threads. Once we have the lock, we set shutdownFlag to inform the other
// threads that the socket is closed.
- rwLock.AcquireWriterLock(-1);
- shutdownFlag = true;
- udpSocket.Close();
- rwLock.ReleaseWriterLock();
-
- // wait for any pending operations to complete on other
- // threads before exiting.
- const int FORCE_STOP = 100;
- int i = 0;
- while (rwOperationCount > 0 && i < FORCE_STOP)
- {
- Thread.Sleep(10);
- ++i;
- }
-
- if (i >= FORCE_STOP)
- {
- Logger.Log("UDPBase.Stop() forced shutdown while waiting on pending operations",
- Helpers.LogLevel.Warning);
- }
+ m_shutdownFlag = true;
+ m_udpSocket.Close();
}
}
- /// <summary>
- ///
- /// </summary>
- public bool IsRunning
- {
- get { return !shutdownFlag; }
- }
-
private void AsyncBeginReceive()
{
- // this method actually kicks off the async read on the socket.
- // we aquire a reader lock here to ensure that no other thread
- // is trying to set shutdownFlag and close the socket.
- rwLock.AcquireReaderLock(-1);
+ // allocate a packet buffer
+ //WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
+ UDPPacketBuffer buf = new UDPPacketBuffer();
- if (!shutdownFlag)
+ if (!m_shutdownFlag)
{
- // increment the count of pending operations
- Interlocked.Increment(ref rwOperationCount);
-
- // allocate a packet buffer
- //WrappedObject<UDPPacketBuffer> wrappedBuffer = Pool.CheckOut();
- UDPPacketBuffer buf = new UDPPacketBuffer();
-
try
{
// kick off an async read
- udpSocket.BeginReceiveFrom(
+ m_udpSocket.BeginReceiveFrom(
//wrappedBuffer.Instance.Data,
buf.Data,
0,
UDPPacketBuffer.BUFFER_SIZE,
SocketFlags.None,
- //ref wrappedBuffer.Instance.RemoteEndPoint,
ref buf.RemoteEndPoint,
- new AsyncCallback(AsyncEndReceive),
+ AsyncEndReceive,
//wrappedBuffer);
buf);
}
- catch (SocketException)
+ catch (SocketException e)
{
- // something bad happened
- //Logger.Log(
- // "A SocketException occurred in UDPServer.AsyncBeginReceive()",
- // Helpers.LogLevel.Error, se);
-
- // an error occurred, therefore the operation is void. Decrement the reference count.
- Interlocked.Decrement(ref rwOperationCount);
+ if (e.SocketErrorCode == SocketError.ConnectionReset)
+ {
+ m_log.Warn("[UDPBASE]: SIO_UDP_CONNRESET was ignored, attempting to salvage the UDP listener on port " + m_udpPort);
+ bool salvaged = false;
+ while (!salvaged)
+ {
+ try
+ {
+ m_udpSocket.BeginReceiveFrom(
+ //wrappedBuffer.Instance.Data,
+ buf.Data,
+ 0,
+ UDPPacketBuffer.BUFFER_SIZE,
+ SocketFlags.None,
+ ref buf.RemoteEndPoint,
+ AsyncEndReceive,
+ //wrappedBuffer);
+ buf);
+ salvaged = true;
+ }
+ catch (SocketException) { }
+ catch (ObjectDisposedException) { return; }
+ }
+
+ m_log.Warn("[UDPBASE]: Salvaged the UDP listener on port " + m_udpPort);
+ }
}
+ catch (ObjectDisposedException) { }
}
-
- // we're done with the socket for now, release the reader lock.
- rwLock.ReleaseReaderLock();
}
private void AsyncEndReceive(IAsyncResult iar)
{
// Asynchronous receive operations will complete here through the call
// to AsyncBeginReceive
-
- // aquire a reader lock
- rwLock.AcquireReaderLock(-1);
-
- if (!shutdownFlag)
+ if (!m_shutdownFlag)
{
+ // start another receive - this keeps the server going!
+ AsyncBeginReceive();
+
// get the buffer that was created in AsyncBeginReceive
// this is the received data
//WrappedObject<UDPPacketBuffer> wrappedBuffer = (WrappedObject<UDPPacketBuffer>)iar.AsyncState;
@@ -241,100 +211,49 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
// get the length of data actually read from the socket, store it with the
// buffer
- buffer.DataLength = udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
-
- // this operation is now complete, decrement the reference count
- Interlocked.Decrement(ref rwOperationCount);
-
- // we're done with the socket, release the reader lock
- rwLock.ReleaseReaderLock();
+ buffer.DataLength = m_udpSocket.EndReceiveFrom(iar, ref buffer.RemoteEndPoint);
// call the abstract method PacketReceived(), passing the buffer that
// has just been filled from the socket read.
PacketReceived(buffer);
}
- catch (SocketException)
- {
- // an error occurred, therefore the operation is void. Decrement the reference count.
- Interlocked.Decrement(ref rwOperationCount);
-
- // we're done with the socket for now, release the reader lock.
- rwLock.ReleaseReaderLock();
- }
- finally
- {
- // start another receive - this keeps the server going!
- AsyncBeginReceive();
-
- //wrappedBuffer.Dispose();
- }
- }
- else
- {
- // nothing bad happened, but we are done with the operation
- // decrement the reference count and release the reader lock
- Interlocked.Decrement(ref rwOperationCount);
- rwLock.ReleaseReaderLock();
+ catch (SocketException) { }
+ catch (ObjectDisposedException) { }
+ //finally { wrappedBuffer.Dispose(); }
}
}
public void AsyncBeginSend(UDPPacketBuffer buf)
{
- rwLock.AcquireReaderLock(-1);
-
- if (!shutdownFlag)
+ if (!m_shutdownFlag)
{
try
{
- Interlocked.Increment(ref rwOperationCount);
- udpSocket.BeginSendTo(
+ m_udpSocket.BeginSendTo(
buf.Data,
0,
buf.DataLength,
SocketFlags.None,
buf.RemoteEndPoint,
- new AsyncCallback(AsyncEndSend),
+ AsyncEndSend,
buf);
}
- catch (SocketException)
- {
- //Logger.Log(
- // "A SocketException occurred in UDPServer.AsyncBeginSend()",
- // Helpers.LogLevel.Error, se);
- }
+ catch (SocketException) { }
+ catch (ObjectDisposedException) { }
}
-
- rwLock.ReleaseReaderLock();
}
- private void AsyncEndSend(IAsyncResult iar)
+ void AsyncEndSend(IAsyncResult result)
{
- rwLock.AcquireReaderLock(-1);
-
- if (!shutdownFlag)
+ try
{
- UDPPacketBuffer buffer = (UDPPacketBuffer)iar.AsyncState;
+ UDPPacketBuffer buf = (UDPPacketBuffer)result.AsyncState;
+ int bytesSent = m_udpSocket.EndSendTo(result);
- try
- {
- int bytesSent = udpSocket.EndSendTo(iar);
-
- // note that call to the abstract PacketSent() method - we are passing the number
- // of bytes sent in a separate parameter, since we can't use buffer.DataLength which
- // is the number of bytes to send (or bytes received depending upon whether this
- // buffer was part of a send or a receive).
- PacketSent(buffer, bytesSent);
- }
- catch (SocketException)
- {
- //Logger.Log(
- // "A SocketException occurred in UDPServer.AsyncEndSend()",
- // Helpers.LogLevel.Error, se);
- }
+ PacketSent(buf, bytesSent);
}
-
- Interlocked.Decrement(ref rwOperationCount);
- rwLock.ReleaseReaderLock();
+ catch (SocketException) { }
+ catch (ObjectDisposedException) { }
}
}
}
--
cgit v1.1