1 module collector; 2 3 import core.thread; 4 import core.sync.mutex; 5 import std.socket; 6 import std.net.curl; 7 import std.json; 8 import std.conv; 9 import std.stdio; 10 import libyggdrasil.libyggdrasil; 11 import std.string; 12 13 14 public final class Collector : Thread 15 { 16 private Address controlYgg; 17 18 private string[] keys; 19 private Mutex keysLock; 20 21 private NodeInfo[string] nodeInfos; 22 private Mutex nodeInfosLock; 23 24 private ulong[string] platforms; 25 private ulong[string] archs; 26 private ulong[string] versions; 27 private Mutex popularityContestLock; 28 29 this(Address controlYgg) 30 { 31 /* Set the worker function */ 32 super(&worker); 33 34 /* Set the address of he admin socket */ 35 this.controlYgg = controlYgg; 36 37 /* Initialize the mutexes (lone star) */ 38 initMutexas(); 39 } 40 41 private void worker() 42 { 43 while(true) 44 { 45 /* Refresh the PeerDB */ 46 refreshKeyDB(); 47 48 /* Refresh PeerInfoDB */ 49 //refreshPeerInfoDB(); 50 51 /* Refresh BuildDB */ 52 refreshBuildDB(); 53 54 writeln("Cycle done"); 55 56 sleep(dur!("seconds")(500)); 57 } 58 } 59 60 private NodeInfo fetchNodeInfo(string key) 61 { 62 /* Get the control socket */ 63 YggdrasilPeer peer = new YggdrasilPeer(controlYgg); 64 65 /* Get the Yggdrasil Node */ 66 YggdrasilNode node = peer.fetchNode(key); 67 68 /* Fetch the NodeInfo */ 69 return node.getNodeInfo(); 70 } 71 72 private void refreshPeerInfoDB() 73 { 74 /* Get all the keys */ 75 string[] keys = getKeys(); 76 77 /* Get the control socket */ 78 YggdrasilPeer peer = new YggdrasilPeer(controlYgg); 79 80 /* Fetch all YggdrasilNode's */ 81 YggdrasilNode[] nodes; 82 for(ulong i = 0; i < keys.length; i++) 83 { 84 /* The current key */ 85 string key = keys[i]; 86 writeln(key); 87 writeln("PeerInfoDB: Fetching node "~to!(string)(i+1)~"/"~to!(string)(keys.length)); 88 nodes ~= peer.fetchNode(key); 89 } 90 91 /* Lock the list */ 92 nodeInfosLock.lock(); 93 94 /* Fetch all node infos and store them */ 95 foreach(YggdrasilNode node; nodes) 96 { 97 nodeInfos[node.getKey()] = node.getNodeInfo(); 98 } 99 100 /* Unlock the list */ 101 nodeInfosLock.unlock(); 102 } 103 104 private bool isKey(ulong[string] hashMap, string key) 105 { 106 foreach(string cKey; hashMap.keys) 107 { 108 if(cmp(cKey, key) == 0) 109 { 110 return true; 111 } 112 } 113 114 return false; 115 } 116 117 public string[] getPlatforms() 118 { 119 popularityContestLock.lock(); 120 string[] platforms = platforms.keys; 121 popularityContestLock.unlock(); 122 return platforms; 123 } 124 125 public ulong getPlatformCount(string key) 126 { 127 ulong count = 0; 128 129 popularityContestLock.lock(); 130 if(key in platforms) 131 { 132 count = platforms[key]; 133 } 134 popularityContestLock.unlock(); 135 136 return count; 137 } 138 139 public string[] getArchs() 140 { 141 popularityContestLock.lock(); 142 string[] archs = archs.keys; 143 popularityContestLock.unlock(); 144 return archs; 145 } 146 147 public ulong getArchCount(string key) 148 { 149 ulong count = 0; 150 151 popularityContestLock.lock(); 152 if(key in archs) 153 { 154 count = archs[key]; 155 } 156 popularityContestLock.unlock(); 157 158 return count; 159 } 160 161 public string[] getVersions() 162 { 163 popularityContestLock.lock(); 164 string[] versions = versions.keys; 165 popularityContestLock.unlock(); 166 return versions; 167 } 168 169 public ulong getVersionCount(string key) 170 { 171 ulong count = 0; 172 173 popularityContestLock.lock(); 174 if(key in versions) 175 { 176 count = versions[key]; 177 } 178 popularityContestLock.unlock(); 179 180 return count; 181 } 182 183 184 185 186 private void refreshBuildDB() 187 { 188 /* Reset stats */ 189 archs = null; 190 platforms = null; 191 versions = null; 192 193 /* Get all the keys */ 194 string[] keys = getKeys(); 195 196 /* Get the control socket */ 197 YggdrasilPeer peer = new YggdrasilPeer(controlYgg); 198 199 /* Fetch all YggdrasilNode's */ 200 YggdrasilNode[] nodes; 201 for(ulong i = 0; i < keys.length; i++) 202 { 203 /* The current key */ 204 string key = keys[i]; 205 writeln(key); 206 writeln("BuildDB: Generating node "~to!(string)(i+1)~"/"~to!(string)(keys.length)); 207 nodes ~= peer.fetchNode(key); 208 } 209 210 ulong i = 0; 211 foreach(YggdrasilNode node; nodes) 212 { 213 writeln("BuildDB: Fetching node information "~to!(string)(i+1)~"/"~to!(string)(nodes.length)); 214 NodeInfo nodeInfo = getNodeInfo(node.getKey()); 215 216 string arch = "unknown"; 217 string os = "unknown"; 218 string _version = "unknown"; 219 220 popularityContestLock.lock(); 221 222 /* If the NodeInfo fetch was successful */ 223 if(nodeInfo) 224 { 225 226 227 /* Whoever runs 221:c99a:91a1:cd2c:3164:27d7:9675:bf7d is a cunt 228 * fucking no build info and i spent this much time, missing key crash 229 */ 230 try 231 { 232 JSONValue nodeInfoJSON = nodeInfo.getFullJSON(); 233 234 /* Fetch information */ 235 arch = nodeInfoJSON["buildarch"].str(); 236 os = nodeInfoJSON["buildplatform"].str(); 237 _version = nodeInfoJSON["buildversion"].str(); 238 239 } 240 catch(JSONException e) 241 { 242 writeln("refreshBuildDB(): Missing some build info"); 243 } 244 } 245 else 246 { 247 writeln("refreshBuildDB(): Missing nodeinfo (timed out)"); 248 arch = "failed"; 249 os = "failed"; 250 _version = "failed"; 251 } 252 253 if(!isKey(archs, arch)) 254 { 255 archs[arch] = 0; 256 } 257 if(!isKey(platforms, os)) 258 { 259 platforms[os] = 0; 260 } 261 if(!isKey(versions, _version)) 262 { 263 versions[_version] = 0; 264 } 265 266 archs[arch]++; 267 platforms[os]++; 268 versions[_version]++; 269 270 popularityContestLock.unlock(); 271 272 i++; 273 } 274 } 275 276 private void refreshKeyDB() 277 { 278 keysLock.lock(); 279 280 string url = "http://[21e:e795:8e82:a9e2:ff48:952d:55f2:f0bb]/static/current"; 281 282 import std.net.curl; 283 string k = cast(string)get(url); 284 285 JSONValue json = parseJSON(k); 286 json = json["yggnodes"]; 287 288 keys = json.object().keys; 289 290 keysLock.unlock(); 291 } 292 293 private void initMutexas() 294 { 295 keysLock = new Mutex(); 296 nodeInfosLock = new Mutex(); 297 popularityContestLock = new Mutex(); 298 } 299 300 public NodeInfo getNodeInfo(string key) 301 { 302 303 // writeln("getNodeInfo()"); 304 NodeInfo nodeInfo; 305 306 /* Lock the list */ 307 nodeInfosLock.lock(); 308 309 /* Find Node (if exists) */ 310 foreach(NodeInfo nodeInfo; nodeInfos) 311 { 312 if(cmp(nodeInfo.getKey(), key) == 0) 313 { 314 nodeInfosLock.unlock(); 315 return nodeInfos[key]; 316 } 317 } 318 319 nodeInfosLock.unlock(); 320 321 nodeInfo = fetchNodeInfo(key); 322 323 nodeInfosLock.lock(); 324 325 if(nodeInfo) 326 { 327 nodeInfos[key] = nodeInfo; 328 } 329 330 331 nodeInfosLock.unlock(); 332 333 return nodeInfo; 334 } 335 336 public string[] getKeys() 337 { 338 string[] copiedKeys; 339 keysLock.lock(); 340 341 foreach(string key; keys) 342 { 343 copiedKeys ~= key; 344 } 345 346 keysLock.unlock(); 347 348 return copiedKeys; 349 } 350 351 }