diff options
-rw-r--r-- | OpenSim/Region/OptionalModules/Framework/Monitoring/ServerStats.cs | 210 |
1 files changed, 50 insertions, 160 deletions
diff --git a/OpenSim/Region/OptionalModules/Framework/Monitoring/ServerStats.cs b/OpenSim/Region/OptionalModules/Framework/Monitoring/ServerStats.cs index 8f60c8d..a3d2436 100644 --- a/OpenSim/Region/OptionalModules/Framework/Monitoring/ServerStats.cs +++ b/OpenSim/Region/OptionalModules/Framework/Monitoring/ServerStats.cs | |||
@@ -63,6 +63,7 @@ public class ServerStats : ISharedRegionModule | |||
63 | public readonly string ContainerNetwork = "network"; | 63 | public readonly string ContainerNetwork = "network"; |
64 | public readonly string ContainerProcess = "process"; | 64 | public readonly string ContainerProcess = "process"; |
65 | 65 | ||
66 | public string NetworkInterfaceTypes = "Ethernet"; | ||
66 | 67 | ||
67 | readonly int performanceCounterSampleInterval = 500; | 68 | readonly int performanceCounterSampleInterval = 500; |
68 | int lastperformanceCounterSampleTime = 0; | 69 | int lastperformanceCounterSampleTime = 0; |
@@ -86,20 +87,6 @@ public class ServerStats : ISharedRegionModule | |||
86 | 87 | ||
87 | PerfCounterControl processorPercentPerfCounter = null; | 88 | PerfCounterControl processorPercentPerfCounter = null; |
88 | 89 | ||
89 | PerfCounterControl processThreadCountPerfCounter = null; | ||
90 | PerfCounterControl processVirtualBytesPerfCounter = null; | ||
91 | PerfCounterControl processWorkingSetPerfCounter = null; | ||
92 | |||
93 | PerfCounterControl dotNETCLRMemoryAllocatedBytesPerSecPerfCounter = null; | ||
94 | PerfCounterControl dotNETCLRMemoryGen0HeapSizePerfCounter = null; | ||
95 | PerfCounterControl dotNETCLRMemoryGen1HeapSizePerfCounter = null; | ||
96 | PerfCounterControl dotNETCLRMemoryGen2HeapSizePerfCounter = null; | ||
97 | |||
98 | PerfCounterControl dotNETCLRLaTTotalContentionsPerfCounter = null; | ||
99 | PerfCounterControl dotNETCLRLaTContentionsPerSecPerfCounter = null; | ||
100 | PerfCounterControl dotNETCLRLaTLogicalThreadsPerfCounter = null; | ||
101 | PerfCounterControl dotNETCLRLaTPhysicalThreadsPerfCounter = null; | ||
102 | |||
103 | #region ISharedRegionModule | 90 | #region ISharedRegionModule |
104 | // IRegionModuleBase.Name | 91 | // IRegionModuleBase.Name |
105 | public string Name { get { return "Server Stats"; } } | 92 | public string Name { get { return "Server Stats"; } } |
@@ -108,10 +95,15 @@ public class ServerStats : ISharedRegionModule | |||
108 | // IRegionModuleBase.Initialize | 95 | // IRegionModuleBase.Initialize |
109 | public void Initialise(IConfigSource source) | 96 | public void Initialise(IConfigSource source) |
110 | { | 97 | { |
111 | IConfig cnfg = source.Configs["Statistics"]; | 98 | IConfig cfg = source.Configs["Monitoring"]; |
99 | |||
100 | if (cfg != null) | ||
101 | Enabled = cfg.GetBoolean("ServerStatsEnabled", true); | ||
112 | 102 | ||
113 | if (cnfg != null) | 103 | if (Enabled) |
114 | Enabled = cnfg.GetBoolean("Enabled", true); | 104 | { |
105 | NetworkInterfaceTypes = cfg.GetString("NetworkInterfaceTypes", "Ethernet"); | ||
106 | } | ||
115 | } | 107 | } |
116 | // IRegionModuleBase.Close | 108 | // IRegionModuleBase.Close |
117 | public void Close() | 109 | public void Close() |
@@ -148,6 +140,13 @@ public class ServerStats : ISharedRegionModule | |||
148 | } | 140 | } |
149 | #endregion ISharedRegionModule | 141 | #endregion ISharedRegionModule |
150 | 142 | ||
143 | private void MakeStat(string pName, string pUnit, string pContainer, Action<Stat> act) | ||
144 | { | ||
145 | Stat stat = new Stat(pName, pName, "", pUnit, CategoryServer, pContainer, StatType.Pull, act, StatVerbosity.Info); | ||
146 | StatsManager.RegisterStat(stat); | ||
147 | RegisteredStats.Add(pName, stat); | ||
148 | } | ||
149 | |||
151 | public void RegisterServerStats() | 150 | public void RegisterServerStats() |
152 | { | 151 | { |
153 | lastperformanceCounterSampleTime = Util.EnvironmentTickCount(); | 152 | lastperformanceCounterSampleTime = Util.EnvironmentTickCount(); |
@@ -157,7 +156,7 @@ public class ServerStats : ISharedRegionModule | |||
157 | 156 | ||
158 | try | 157 | try |
159 | { | 158 | { |
160 | tempName = "CPU_Percent"; | 159 | tempName = "CPUPercent"; |
161 | tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total"); | 160 | tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total"); |
162 | processorPercentPerfCounter = new PerfCounterControl(tempPC); | 161 | processorPercentPerfCounter = new PerfCounterControl(tempPC); |
163 | // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. | 162 | // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. |
@@ -167,31 +166,17 @@ public class ServerStats : ISharedRegionModule | |||
167 | StatsManager.RegisterStat(tempStat); | 166 | StatsManager.RegisterStat(tempStat); |
168 | RegisteredStats.Add(tempName, tempStat); | 167 | RegisteredStats.Add(tempName, tempStat); |
169 | 168 | ||
170 | /* Performance counters are not the way to go. Ick. Find another way. | 169 | MakeStat("TotalProcessorTime", "sec", ContainerProcessor, |
171 | tempName = "Thread_Count"; | 170 | (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; }); |
172 | tempPC = new PerformanceCounter("Process", "Thread Count", AppDomain.CurrentDomain.FriendlyName); | ||
173 | processThreadCountPerfCounter = new PerfCounterControl(tempPC); | ||
174 | tempStat = new Stat("Thread_Count", "Thread_Count", "", "threads", CategoryServer, ContainerProcess, | ||
175 | StatType.Pull, (s) => { GetNextValue(s, processThreadCountPerfCounter); }, StatVerbosity.Info); | ||
176 | StatsManager.RegisterStat(tempStat); | ||
177 | RegisteredStats.Add(tempName, tempStat); | ||
178 | 171 | ||
179 | tempName = "Virtual_Bytes"; | 172 | MakeStat("UserProcessorTime", "sec", ContainerProcessor, |
180 | tempPC = new PerformanceCounter("Process", "Virtual Bytes", AppDomain.CurrentDomain.FriendlyName); | 173 | (s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; }); |
181 | processVirtualBytesPerfCounter = new PerfCounterControl(tempPC); | ||
182 | tempStat = new Stat("Virtual_Bytes", "Virtual_Bytes", "", "MB", CategoryServer, ContainerProcess, | ||
183 | StatType.Pull, (s) => { GetNextValue(s, processVirtualBytesPerfCounter, 1024.0*1024.0); }, StatVerbosity.Info); | ||
184 | StatsManager.RegisterStat(tempStat); | ||
185 | RegisteredStats.Add(tempName, tempStat); | ||
186 | 174 | ||
187 | tempName = "Working_Set"; | 175 | MakeStat("PrivilegedProcessorTime", "sec", ContainerProcessor, |
188 | tempPC = new PerformanceCounter("Process", "Working Set", AppDomain.CurrentDomain.FriendlyName); | 176 | (s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; }); |
189 | processWorkingSetPerfCounter = new PerfCounterControl(tempPC); | 177 | |
190 | tempStat = new Stat("Working_Set", "Working_Set", "", "MB", CategoryServer, ContainerProcess, | 178 | MakeStat("Threads", "threads", ContainerProcessor, |
191 | StatType.Pull, (s) => { GetNextValue(s, processWorkingSetPerfCounter, 1024.0*1024.0); }, StatVerbosity.Info); | 179 | (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; }); |
192 | StatsManager.RegisterStat(tempStat); | ||
193 | RegisteredStats.Add(tempName, tempStat); | ||
194 | */ | ||
195 | } | 180 | } |
196 | catch (Exception e) | 181 | catch (Exception e) |
197 | { | 182 | { |
@@ -200,112 +185,33 @@ public class ServerStats : ISharedRegionModule | |||
200 | 185 | ||
201 | try | 186 | try |
202 | { | 187 | { |
203 | /* The ".NET CLR *" categories aren't working for me. | 188 | List<string> okInterfaceTypes = new List<string>(NetworkInterfaceTypes.Split(',')); |
204 | tempName = ""Bytes_Allocated_Per_Sec"; | ||
205 | tempPC = new PerformanceCounter(".NET CLR Memory", "Allocated Bytes/sec", AppDomain.CurrentDomain.FriendlyName); | ||
206 | dotNETCLRMemoryAllocatedBytesPerSecPerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
207 | tempStat = new Stat(tempName, tempName, "", "bytes/sec", ServerCategory, MemoryContainer, | ||
208 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRMemoryAllocatedBytesPerSecPerfCounter); }, StatVerbosity.Info); | ||
209 | StatsManager.RegisterStat(tempStat); | ||
210 | RegisteredStats.Add(tempName, tempStat); | ||
211 | |||
212 | tempName = "Gen_0_Heap_Size"; | ||
213 | tempPC = new PerformanceCounter(".NET CLR Memory", "Gen 0 heap size", AppDomain.CurrentDomain.FriendlyName); | ||
214 | dotNETCLRMemoryGen0HeapSizePerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
215 | tempStat = new Stat("Gen_0_Heap_Size", "Gen_0_Heap_Size", "", "bytes", ServerCategory, MemoryContainer, | ||
216 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRMemoryGen0HeapSizePerfCounter); }, StatVerbosity.Info); | ||
217 | StatsManager.RegisterStat(tempStat); | ||
218 | RegisteredStats.Add(tempName, tempStat); | ||
219 | |||
220 | tempName = "Gen_1_Heap_Size"; | ||
221 | tempPC = new PerformanceCounter(".NET CLR Memory", "Gen 1 heap size", AppDomain.CurrentDomain.FriendlyName); | ||
222 | dotNETCLRMemoryGen1HeapSizePerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
223 | tempStat = new Stat("Gen_1_Heap_Size", "Gen_1_Heap_Size", "", "bytes", ServerCategory, MemoryContainer, | ||
224 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRMemoryGen1HeapSizePerfCounter); }, StatVerbosity.Info); | ||
225 | StatsManager.RegisterStat(tempStat); | ||
226 | RegisteredStats.Add(tempName, tempStat); | ||
227 | |||
228 | tempName = "Gen_2_Heap_Size"; | ||
229 | tempPC = new PerformanceCounter(".NET CLR Memory", "Gen 2 heap size", AppDomain.CurrentDomain.FriendlyName); | ||
230 | dotNETCLRMemoryGen2HeapSizePerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
231 | tempStat = new Stat("Gen_2_Heap_Size", "Gen_2_Heap_Size", "", "bytes", ServerCategory, MemoryContainer, | ||
232 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRMemoryGen2HeapSizePerfCounter); }, StatVerbosity.Info); | ||
233 | StatsManager.RegisterStat(tempStat); | ||
234 | RegisteredStats.Add(tempName, tempStat); | ||
235 | |||
236 | tempName = "Total_Lock_Contentions"; | ||
237 | tempPC = new PerformanceCounter(".NET CLR LocksAndThreads", "Total # of Contentions"); | ||
238 | dotNETCLRLaTTotalContentionsPerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
239 | tempStat = new Stat("Total_Lock_Contentions", "Total_Lock_Contentions", "", "contentions", ServerCategory, ProcessContainer, | ||
240 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRLaTTotalContentionsPerfCounter); }, StatVerbosity.Info); | ||
241 | StatsManager.RegisterStat(tempStat); | ||
242 | RegisteredStats.Add(tempName, tempStat); | ||
243 | |||
244 | tempName = "Lock_Contentions"; | ||
245 | tempPC = new PerformanceCounter(".NET CLR LocksAndThreads", "Contention Rate / sec"); | ||
246 | dotNETCLRLaTContentionsPerSecPerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
247 | tempStat = new Stat("Lock_Contentions", "Lock_Contentions", "", "contentions/sec", ServerCategory, ProcessContainer, | ||
248 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRLaTContentionsPerSecPerfCounter); }, StatVerbosity.Info); | ||
249 | StatsManager.RegisterStat(tempStat); | ||
250 | RegisteredStats.Add(tempName, tempStat); | ||
251 | |||
252 | tempName = "Logical_Threads"; | ||
253 | tempPC = new PerformanceCounter(".NET CLR LocksAndThreads", "# of current logical Threads"); | ||
254 | dotNETCLRLaTLogicalThreadsPerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
255 | tempStat = new Stat("Logicial_Threads", "Logicial_Threads", "", "threads", ServerCategory, ProcessContainer, | ||
256 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRLaTLogicalThreadsPerfCounter); }, StatVerbosity.Info); | ||
257 | StatsManager.RegisterStat(tempStat); | ||
258 | RegisteredStats.Add(tempName, tempStat); | ||
259 | |||
260 | tempName = "Physical_Threads"; | ||
261 | tempPC = new PerformanceCounter(".NET CLR LocksAndThreads", "# of current physical Threads"); | ||
262 | dotNETCLRLaTPhysicalThreadsPerfCounter = new PerfCounterControl(tempPC, tempStat); | ||
263 | tempStat = new Stat("Physical_Threads", "Physical_Threads", "", "threads", ServerCategory, ProcessContainer, | ||
264 | StatType.Pull, (s) => { GetNextValue(s, dotNETCLRLaTPhysicalThreadsPerfCounter); }, StatVerbosity.Info); | ||
265 | StatsManager.RegisterStat(tempStat); | ||
266 | RegisteredStats.Add(tempName, tempStat); | ||
267 | */ | ||
268 | } | ||
269 | catch (Exception e) | ||
270 | { | ||
271 | m_log.ErrorFormat("{0} Exception creating '.NET CLR Memory': {1}", LogHeader, e); | ||
272 | } | ||
273 | 189 | ||
274 | try | ||
275 | { | ||
276 | IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces(); | 190 | IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces(); |
277 | // IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces().Where( | ||
278 | // (network) => network.NetworkInterfaceType == NetworkInterfaceType.Ethernet); | ||
279 | // IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces().Where( | ||
280 | // (network) => network.OperationalStatus == OperationalStatus.Up); | ||
281 | |||
282 | foreach (NetworkInterface nic in nics) | 191 | foreach (NetworkInterface nic in nics) |
283 | { | 192 | { |
284 | if (nic.OperationalStatus != OperationalStatus.Up || nic.NetworkInterfaceType != NetworkInterfaceType.Ethernet) | 193 | if (nic.OperationalStatus != OperationalStatus.Up) |
194 | continue; | ||
195 | |||
196 | string nicInterfaceType = nic.NetworkInterfaceType.ToString(); | ||
197 | if (!okInterfaceTypes.Contains(nicInterfaceType)) | ||
198 | { | ||
199 | m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'. To include, add to [Monitoring]NetworkInterfaceTypes='Ethernet,Loopback'", | ||
200 | LogHeader, nic.Name, nicInterfaceType); | ||
285 | continue; | 201 | continue; |
202 | } | ||
286 | 203 | ||
287 | if (nic.Supports(NetworkInterfaceComponent.IPv4)) | 204 | if (nic.Supports(NetworkInterfaceComponent.IPv4)) |
288 | { | 205 | { |
289 | IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics(); | 206 | IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics(); |
290 | if (nicStats != null) | 207 | if (nicStats != null) |
291 | { | 208 | { |
292 | tempName = "Bytes_Rcvd/" + nic.Name; | 209 | MakeStat("BytesRcvd/" + nic.Name, "KB", ContainerNetwork, |
293 | tempStat = new Stat(tempName, tempName, nic.Name, "KB", CategoryServer, ContainerNetwork, | 210 | (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); }); |
294 | StatType.Pull, (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); }, StatVerbosity.Info); | 211 | MakeStat("BytesSent/" + nic.Name, "KB", ContainerNetwork, |
295 | StatsManager.RegisterStat(tempStat); | 212 | (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); }); |
296 | RegisteredStats.Add(tempName, tempStat); | 213 | MakeStat("TotalBytes/" + nic.Name, "KB", ContainerNetwork, |
297 | 214 | (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); }); | |
298 | tempName = "Bytes_Sent/" + nic.Name; | ||
299 | tempStat = new Stat(tempName, tempName, nic.Name, "KB", CategoryServer, ContainerNetwork, | ||
300 | StatType.Pull, (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); }, StatVerbosity.Info); | ||
301 | StatsManager.RegisterStat(tempStat); | ||
302 | RegisteredStats.Add(tempName, tempStat); | ||
303 | |||
304 | tempName = "Total_Bytes/" + nic.Name; | ||
305 | tempStat = new Stat(tempName, tempName, nic.Name, "KB", CategoryServer, ContainerNetwork, | ||
306 | StatType.Pull, (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); }, StatVerbosity.Info); | ||
307 | StatsManager.RegisterStat(tempStat); | ||
308 | RegisteredStats.Add(tempName, tempStat); | ||
309 | } | 215 | } |
310 | } | 216 | } |
311 | } | 217 | } |
@@ -315,30 +221,14 @@ public class ServerStats : ISharedRegionModule | |||
315 | m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e); | 221 | m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e); |
316 | } | 222 | } |
317 | 223 | ||
318 | tempName = "Process_Memory"; | 224 | MakeStat("ProcessMemory", "MB", ContainerMemory, |
319 | tempStat = new Stat(tempName, tempName, "", "MB", CategoryServer, ContainerMemory, | 225 | (s) => { s.Value = Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d; }); |
320 | StatType.Pull, (s) => { s.Value = Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d; }, StatVerbosity.Info); | 226 | MakeStat("ObjectMemory", "MB", ContainerMemory, |
321 | StatsManager.RegisterStat(tempStat); | 227 | (s) => { s.Value = GC.GetTotalMemory(false) / 1024d / 1024d; }); |
322 | RegisteredStats.Add(tempName, tempStat); | 228 | MakeStat("LastMemoryChurn", "MB/sec", ContainerMemory, |
323 | 229 | (s) => { s.Value = Math.Round(MemoryWatchdog.LastMemoryChurn * 1000d / 1024d / 1024d, 3); }); | |
324 | tempName = "Object_Memory"; | 230 | MakeStat("AverageMemoryChurn", "MB/sec", ContainerMemory, |
325 | tempStat = new Stat(tempName, tempName, "", "MB", CategoryServer, ContainerMemory, | 231 | (s) => { s.Value = Math.Round(MemoryWatchdog.AverageMemoryChurn * 1000d / 1024d / 1024d, 3); }); |
326 | StatType.Pull, (s) => { s.Value = GC.GetTotalMemory(false) / 1024d / 1024d; }, StatVerbosity.Info); | ||
327 | StatsManager.RegisterStat(tempStat); | ||
328 | RegisteredStats.Add(tempName, tempStat); | ||
329 | |||
330 | tempName = "Last_Memory_Churn"; | ||
331 | tempStat = new Stat(tempName, tempName, "", "MB/sec", CategoryServer, ContainerMemory, | ||
332 | StatType.Pull, (s) => { s.Value = Math.Round(MemoryWatchdog.LastMemoryChurn * 1000d / 1024d / 1024d, 3); }, StatVerbosity.Info); | ||
333 | StatsManager.RegisterStat(tempStat); | ||
334 | RegisteredStats.Add(tempName, tempStat); | ||
335 | |||
336 | tempName = "Average_Memory_Churn"; | ||
337 | tempStat = new Stat(tempName, tempName, "", "MB/sec", CategoryServer, ContainerMemory, | ||
338 | StatType.Pull, (s) => { s.Value = Math.Round(MemoryWatchdog.AverageMemoryChurn * 1000d / 1024d / 1024d, 3); }, StatVerbosity.Info); | ||
339 | StatsManager.RegisterStat(tempStat); | ||
340 | RegisteredStats.Add(tempName, tempStat); | ||
341 | |||
342 | } | 232 | } |
343 | 233 | ||
344 | // Notes on performance counters: | 234 | // Notes on performance counters: |