1 module collector;
2 
3 import core.thread;
4 import core.sync.mutex;
5 import std.socket;
6 import std.net.curl;
7 import std.json;
8 import std.conv;
9 import std.stdio;
10 import libyggdrasil.libyggdrasil;
11 import std.string;
12 
13 
14 public final class Collector : Thread
15 {
16 	private Address controlYgg;
17 
18 	private string[] keys;
19 	private Mutex keysLock;
20 
21 	private NodeInfo[string] nodeInfos;
22 	private Mutex nodeInfosLock;
23 
24 	private ulong[string] platforms;
25 	private ulong[string] archs;
26 	private ulong[string] versions;
27 	private Mutex popularityContestLock;
28 
29 	this(Address controlYgg)
30 	{
31 		/* Set the worker function */
32 		super(&worker);
33 
34 		/* Set the address of he admin socket */
35 		this.controlYgg = controlYgg;
36 		
37 		/* Initialize the mutexes (lone star) */
38 		initMutexas();
39 	}
40 
41 	private void worker()
42 	{
43 		while(true)
44 		{
45 			/* Refresh the PeerDB */
46 			refreshKeyDB();
47 
48 			/* Refresh PeerInfoDB */
49 			//refreshPeerInfoDB();
50 
51 			/* Refresh BuildDB */
52 			refreshBuildDB();
53 
54 			writeln("Cycle done");
55 			
56 			sleep(dur!("seconds")(500));
57 		}
58 	}
59 
60 	private NodeInfo fetchNodeInfo(string key)
61 	{
62 		/* Get the control socket */
63 		YggdrasilPeer peer = new YggdrasilPeer(controlYgg);
64 
65 		/* Get the Yggdrasil Node */
66 		YggdrasilNode node = peer.fetchNode(key);
67 
68 		/* Fetch the NodeInfo */
69 		return node.getNodeInfo();
70 	}
71 
72 	private void refreshPeerInfoDB()
73 	{
74 		/* Get all the keys */
75 		string[] keys = getKeys();
76 
77 		/* Get the control socket */
78 		YggdrasilPeer peer = new YggdrasilPeer(controlYgg);
79 
80 		/* Fetch all YggdrasilNode's */
81 		YggdrasilNode[] nodes;
82 		for(ulong i = 0; i < keys.length; i++)
83 		{
84 			/* The current key */
85 			string key = keys[i];
86 			writeln(key);
87 			writeln("PeerInfoDB: Fetching node "~to!(string)(i+1)~"/"~to!(string)(keys.length));
88 			nodes ~= peer.fetchNode(key);
89 		}
90 
91 		/* Lock the list */
92 		nodeInfosLock.lock();
93 
94 		/* Fetch all node infos and store them */
95 		foreach(YggdrasilNode node; nodes)
96 		{
97 			nodeInfos[node.getKey()] = node.getNodeInfo();
98 		}
99 
100 		/* Unlock the list */
101 		nodeInfosLock.unlock();
102 	}
103 
104 	private bool isKey(ulong[string] hashMap, string key)
105 	{
106 		foreach(string cKey; hashMap.keys)
107 		{
108 			if(cmp(cKey, key) == 0)
109 			{
110 				return true;
111 			}
112 		}
113 
114 		return false;
115 	}
116 
117 	public string[] getPlatforms()
118 	{
119 		popularityContestLock.lock();
120 		string[] platforms = platforms.keys;
121 		popularityContestLock.unlock();
122 		return platforms;
123 	}
124 
125 	public ulong getPlatformCount(string key)
126 	{
127 		ulong count = 0;
128 
129 		popularityContestLock.lock();
130 		if(key in platforms)
131 		{
132 			count = platforms[key];
133 		}
134 		popularityContestLock.unlock();
135 		
136 		return count;
137 	}
138 
139 	public string[] getArchs()
140 	{
141 		popularityContestLock.lock();
142 		string[] archs = archs.keys;
143 		popularityContestLock.unlock();
144 		return archs;
145 	}
146 
147 	public ulong getArchCount(string key)
148 	{
149 		ulong count = 0;
150 
151 		popularityContestLock.lock();
152 		if(key in archs)
153 		{
154 			count = archs[key];
155 		}
156 		popularityContestLock.unlock();
157 		
158 		return count;
159 	}
160 
161 	public string[] getVersions()
162 	{
163 		popularityContestLock.lock();
164 		string[] versions = versions.keys;
165 		popularityContestLock.unlock();
166 		return versions;
167 	}
168 
169 	public ulong getVersionCount(string key)
170 	{
171 		ulong count = 0;
172 
173 		popularityContestLock.lock();
174 		if(key in versions)
175 		{
176 			count = versions[key];
177 		}
178 		popularityContestLock.unlock();
179 
180 		return count;
181 	}
182 
183 
184 
185 
186 	private void refreshBuildDB()
187 	{
188 		/* Reset stats */
189 		archs = null;
190 		platforms = null;
191 		versions = null;
192 		
193 		/* Get all the keys */
194 		string[] keys = getKeys();
195 
196 		/* Get the control socket */
197 		YggdrasilPeer peer = new YggdrasilPeer(controlYgg);
198 
199 		/* Fetch all YggdrasilNode's */
200 		YggdrasilNode[] nodes;
201 		for(ulong i = 0; i < keys.length; i++)
202 		{
203 			/* The current key */
204 			string key = keys[i];
205 			writeln(key);
206 			writeln("BuildDB: Generating node "~to!(string)(i+1)~"/"~to!(string)(keys.length));
207 			nodes ~= peer.fetchNode(key);
208 		}
209 
210 		ulong i = 0;
211 		foreach(YggdrasilNode node; nodes)
212 		{
213 			writeln("BuildDB: Fetching node information "~to!(string)(i+1)~"/"~to!(string)(nodes.length));
214 			NodeInfo nodeInfo = getNodeInfo(node.getKey());
215 
216 			string arch = "unknown";
217 			string os = "unknown";
218 			string _version = "unknown";
219 
220 			popularityContestLock.lock();
221 			
222 			/* If the NodeInfo fetch was successful */
223 			if(nodeInfo)
224 			{
225 				
226 				
227 				/* Whoever runs 221:c99a:91a1:cd2c:3164:27d7:9675:bf7d is a cunt
228 				* fucking no build info and i spent this much time, missing key crash
229 				*/
230 				try
231 				{
232 					JSONValue nodeInfoJSON = nodeInfo.getFullJSON();
233 				
234 					/* Fetch information */
235 					arch = nodeInfoJSON["buildarch"].str();
236 					os = nodeInfoJSON["buildplatform"].str();
237 					_version = nodeInfoJSON["buildversion"].str();
238 
239 				}
240 				catch(JSONException e)
241 				{
242 					writeln("refreshBuildDB(): Missing some build info");
243 				}
244 			}
245 			else
246 			{
247 				writeln("refreshBuildDB(): Missing nodeinfo (timed out)");
248 				arch = "failed";
249 				os = "failed";
250 				_version = "failed";
251 			}
252 
253 			if(!isKey(archs, arch))
254 			{
255 				archs[arch] = 0;
256 			}
257 			if(!isKey(platforms, os))
258 			{
259 				platforms[os] = 0;
260 			}
261 			if(!isKey(versions, _version))
262 			{
263 				versions[_version] = 0;
264 			}
265 												
266 			archs[arch]++;
267 			platforms[os]++;
268 			versions[_version]++;
269 
270 			popularityContestLock.unlock();
271 
272 			i++;
273 		}
274 	}
275 
276 	private void refreshKeyDB()
277 	{
278 		keysLock.lock();
279 
280 		string url = "http://[21e:e795:8e82:a9e2:ff48:952d:55f2:f0bb]/static/current";
281 			
282 		import std.net.curl;
283 		string k = cast(string)get(url);
284 		
285 		JSONValue json = parseJSON(k);
286 		json = json["yggnodes"];
287 		
288 		keys = json.object().keys;
289 		
290 		keysLock.unlock();
291 	}
292 
293 	private void initMutexas()
294 	{
295 		keysLock = new Mutex();
296 		nodeInfosLock = new Mutex();
297 		popularityContestLock = new Mutex();
298 	}
299 
300 	public NodeInfo getNodeInfo(string key)
301 	{
302 
303 	//	writeln("getNodeInfo()");
304 		NodeInfo nodeInfo;
305 		
306 		/* Lock the list */
307 		nodeInfosLock.lock();
308 
309 		/* Find Node (if exists) */
310 		foreach(NodeInfo nodeInfo; nodeInfos)
311 		{
312 			if(cmp(nodeInfo.getKey(), key) == 0)
313 			{
314 				nodeInfosLock.unlock();
315 				return nodeInfos[key];
316 			}
317 		}
318 
319 		nodeInfosLock.unlock();
320 
321 		nodeInfo = fetchNodeInfo(key);
322 		
323 		nodeInfosLock.lock();
324 
325 		if(nodeInfo)
326 		{
327 			nodeInfos[key] = nodeInfo;	
328 		}
329 		
330 
331 		nodeInfosLock.unlock();
332 
333 		return nodeInfo;
334 	}
335 	
336 	public string[] getKeys()
337 	{
338 		string[] copiedKeys;
339 		keysLock.lock();
340 
341 		foreach(string key; keys)
342 		{
343 			copiedKeys ~= key;
344 		}
345 
346 		keysLock.unlock();
347 
348 		return copiedKeys;
349 	}
350 	
351 }