Initial version

1 files changed, 279 insertions, 0 deletions

diff --git a/gard-v2.py b/gard-v2.py
new file mode 100755
index 0000000..3a623fc
--- /dev/null
+++ b/gard-v2.py
@@ -0,0 +1,279 @@
+#!/usr/bin/python
+# Header   : $Header: /usr/local/cvsroot/shyam/scripts/gard-v2.py,v 1.2 2008/08/27 10:43:01 shyam Exp $
+# Purpose  : Script to automatically check new Gentoo rsync and distfile mirrors
+# Author   : Shyam Mani <fox2mike@gentoo.org>
+# Created  : 05 Jul 2008
+
+# Dedicated to robbat2 aka Robin, for sparking off the idea ;)
+
+import sys
+import time
+import os
+import datetime
+import subprocess
+
+rmaxlag=45
+dmaxlag=5.5
+relmaxlag=2
+
+reportpath="/home/fox2mike/gentoo/mirrors/gard/reports"
+bugurl="http://bugs.gentoo.org/buglist.cgi?query_format=advanced&product=Mirrors&status_whiteboard_type=allwordssubstr&status_whiteboard=due&bug_status=ASSIGNED&columnlist=bug_id,opendate,bug_status,resolution,short_desc,status_whiteboard&ctype=csv"
+fname="/home/fox2mike/open-mirror-bugs"
+
+def mirrorcheck(target):
+	retcode1=subprocess.call(['/usr/bin/wget','-q','-T 30',target])
+	if retcode1 > 0:
+		error1 = "*ERROR* : return value of wget during mirrorcheck was "+str(retcode1)
+		report.write(error1)
+		#report.close()
+		print error1
+		return
+	else:
+		temp=open('timestamp.chk','r')
+		ots=temp.readline()
+		temp.close()
+		#Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python
+		t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S "))
+		lag=(time.mktime(time.gmtime())-t1)/(60*60)
+		gmt=time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
+		if lag > dmaxlag:
+			error1 = "*ERROR* : distfiles is over 5.5 hours behind, lag is : "+str(round(lag,2))+" hours"+'\n'
+			tstamp1 = "Timestamp on Mirror   : "+str(ots)
+			curtime1 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+			bugurl1 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+			report.write(error1)
+			report.write(tstamp1)
+			report.write(curtime1)
+			report.write(bugurl1)
+			#report.close()
+			print error1,tstamp1,curtime1,bugurl1
+		else:
+			info1 = addy+" is in sync, current delta : "+str(round(lag,2))+" hours"+'\n'
+			tstamp1 = "Timestamp on Mirror   : "+str(ots)
+			curtime1 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+			bugurl1 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+			report.write(info1)
+			report.write(tstamp1)
+			report.write(curtime1)
+			report.write(bugurl1)
+			#report.close()
+			print info1,tstamp1,curtime1,bugurl1
+		os.unlink('timestamp.chk')
+	return
+
+def relpermcheck(target):
+	retcode2=subprocess.call(['/usr/bin/wget','-q','-T 30',target])
+	if retcode2 < 1:
+		error2 = "*ERROR* : return value of wget for /releases file perm check was "+str(retcode2)+" which means the file is world accessible!"+'\n'
+		report.write(error2)
+		report.close()
+		print error2
+		os.unlink('THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt')
+	return
+
+def reltscheck(target):
+	retcode3=subprocess.call(['/usr/bin/wget','-q','-T 30',target])
+	if retcode3 > 0:
+		error3 = "*ERROR* : return value of wget for release timestamp check was "+str(retcode3)+'\n'
+		report.write(error3)
+		#report.close()
+		print error3
+		return
+	else:
+		temp=open('timestamp.x','r')
+		ots=temp.readline()
+		temp.close()
+		ts=ots.split(' ')
+		nots=ts[1]+" "+ts[2]+" "+ts[3]+" "+ts[4]+" "+ts[5]+" "
+		#Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python
+		t1=time.mktime(time.strptime(nots, "%a, %d %b %Y %H:%M:%S "))
+		lag=(time.mktime(time.gmtime())-t1)/(60*60*24)
+		gmt=time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
+		if lag > relmaxlag:
+			error3 = "*ERROR* : releases is over 2 days behind, lag is : "+str(round(lag,2))+" days"+'\n'
+			tstamp3 = "Timestamp on Mirror   : "+nots+'\n'
+			curtime3 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+			bugurl3 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+			report.write(error3)
+			report.write(tstamp3)
+			report.write(curtime3)
+			report.write(bugurl3)
+			#report.close()
+			print error3,tstamp3,curtime3,bugurl3
+		else:
+			info3 = addy+"'s releases dir is in sync, current delta : "+str(round(lag,2))+" days"+'\n'
+			tstamp3 = "Timestamp on Mirror   : "+nots+'\n'
+			curtime3 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+			bugurl3 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+			report.write(info3)
+			report.write(tstamp3)
+			report.write(curtime3)
+			report.write(bugurl3)
+			#report.close()
+			print info3,tstamp3,curtime3,bugurl3
+		os.unlink('timestamp.x')
+	return
+
+
+retcode5=subprocess.call(['/usr/bin/wget','-q','-T 30', '-O', fname, bugurl])
+file=open(fname,'r')
+print "\nInitializing G(entoo)A(utomated)R(sync)D(istfiles) checker script, version 2 (Katrina) - kicking mirror ass for Gentoo \n"
+
+# Format of CSV = 234083,"2008-08-06 13:01:32","ASSIGNED",,"[rsync,distfiles|all] New mirror Cambrium BV (The Netherlands)","due:2008/09/01;dist:http://mirror.cambrium.nl/pub/os/linux/gentoo/|ftp://mirror.cambrium.nl/pub/os/linux/gentoo/|rsync://mirror.cambrium.nl/gentoo/;portage:mirror.cambrium.nl"
+
+for line in file:
+	line=line.replace("\n","")
+	data=line.split(',')
+	bugnum=data[0]
+	swboard=data[5]
+	if bugnum == "bug_id":
+		continue
+	else:
+		details=swboard.split(';')
+		duedate=details[0]
+		duedate=duedate.replace("\"","")
+		#print duedate
+		urls=details[1].replace("dist:","")
+		port=details[2].split(':')
+		grsync=port[1].replace("\"","")
+		temp=urls.split('|')
+		http=temp[0]
+		ftp=temp[1]
+		drsync=temp[2]
+		#print http
+		#print ftp
+		#print drsync
+		#print grsync
+		if len(http) > 0:
+			boo=http.split('/')
+			addy=boo[2]
+		elif len(ftp) > 0:
+			boo=ftp.split('/')
+			addy=boo[2]
+		else:
+			addy=grsync
+		#hdomain='/'+addy[2]
+		if len(grsync) > 0:
+			fname=reportpath+"/"+grsync
+			if os.path.isdir(reportpath):
+				pass
+	        	else:
+        	        	os.makedirs(reportpath)
+			report=open(fname,'a+')
+			info4=grsync+" wants to be a gentoo-portage mirror and is being checked as per Bug #"+bugnum+'\n'
+			report.write(info4)
+			print info4
+			target=grsync+"::gentoo-portage/metadata/timestamp.chk"
+			retcode=subprocess.call(['rsync','-aqP','--no-motd','--contimeout=30',target,'.'])
+			if retcode > 0:
+				error4="*ERROR* : return value of rsync during gentoo-portage check was "+str(retcode)
+				report.write(error4)
+				report.close()
+				print error4
+			else:	
+				temp=open('timestamp.chk','r')
+				ots=temp.readline()
+				temp.close()
+				#Keep timestamp as Fri, 04 Jul 2008 20:45:01, remove the +0000 since there doesn't seem to be a %z in python
+				t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S "))
+				lag=(time.mktime(time.gmtime())-t1)/60
+				if lag > rmaxlag:
+					error4 = "*ERROR* : gentoo-portage mirror is lagging by over 40 mins, lag is : "+str(round(lag,2))+" mins"+'\n'
+					tstamp4 = "Timestamp on Mirror   : "+str(ots)
+					curtime4 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+					bugurl4 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+					report.write(error4)
+					report.write(tstamp4)
+					report.write(curtime4)
+					report.write(bugurl4)
+					report.close()
+					print error4,tstamp4,curtime4,bugurl4
+				else:
+					info4 = grsync+" is in sync, current delta : "+str(round(lag,2))+" mins"+'\n'
+					tstamp4 = "Timestamp on Mirror   : "+str(ots)
+					curtime4= "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+					bugurl4 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+					report.write(info4)
+					report.write(tstamp4)
+					report.write(curtime4)
+					report.write(bugurl4)
+					report.close()
+					print info4,tstamp4,curtime4,bugurl4
+				os.unlink('timestamp.chk')
+		if len(http) > 0:
+			fname=reportpath+"/"+addy
+			if os.path.isdir(reportpath):
+      				pass
+        		else:
+       				os.makedirs(reportpath)
+			report=open(fname,'a+')
+			info5 = addy+" wants to be distfiles mirror and is being checked as per Bug #"+bugnum+ " over http"+'\n'
+			report.write(info5)
+			print info5
+			disttsdest=http+"/distfiles/timestamp.chk"
+			mirrorcheck(disttsdest)
+			reltsdest=http+"/releases/.test/timestamp.x"
+			reltscheck(reltsdest)
+			relpermdest=http+"/releases/.test/THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt"
+			relpermcheck(relpermdest)
+		if len(ftp) > 0:
+			fname=reportpath+"/"+addy
+			if os.path.isdir(reportpath):
+       				pass
+       			else:
+       				os.makedirs(reportpath)
+			report=open(fname,'a+')
+			info6 = addy+" wants to be an distfiles mirror and is being checked as per Bug #"+bugnum+ " over ftp"+'\n'
+			report.write(info6)
+			print info6
+			ftpdest=ftp+"/distfiles/timestamp.chk"
+			mirrorcheck(ftpdest)
+			reltsdest=ftp+"/releases/.test/timestamp.x"
+			reltscheck(reltsdest)
+			relpermdest=ftp+"/releases/.test/THIS-FILE-SHOULD-NOT-BE-PUBLIC.txt"
+			relpermcheck(relpermdest)
+		if len(drsync) > 0:
+			fname=reportpath+"/"+addy
+			if os.path.isdir(reportpath):
+	       			pass
+      			else:
+               			os.makedirs(reportpath)
+			report=open(fname,'a+')
+			info7 = addy+" wants to be an distfiles mirror and is being checked as per Bug #"+bugnum+ " over rsync"+'\n'
+			report.write(info7)
+			print info7
+			target=drsync+"/distfiles/timestamp.chk"
+			retcode4=subprocess.call(['rsync','-aqP','--no-motd','--contimeout=30',target,'.'])
+			if retcode4 > 0:
+				error7 = "*ERROR* : return value of rsync during distfiles check was "+str(retcode4)
+				report.write(error7)
+				print error7
+			else:
+				temp=open('timestamp.chk','r')
+				ots=temp.readline()
+				temp.close()
+				t1=time.mktime(time.strptime(ots.replace(" +0000",""), "%a, %d %b %Y %H:%M:%S "))
+				lag=(time.mktime(time.gmtime())-t1)/(60*60)
+				if lag > dmaxlag:
+					error7 = "*ERROR* : Mirror is lagging by over 5.5 hours, difference is : "+str(round(lag,2))+" hours"+'\n'
+					tstamp7 = "Timestamp on Mirror   : "+str(ots)
+					curtime7= "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+					bugurl7 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+					report.write(error7)
+					report.write(tstamp7)
+					report.write(curtime7)
+					report.write(bugurl7)
+					report.close()
+					print error7,tstamp7,curtime7,bugurl7
+				else:
+					info7 = addy+" is in sync, current lag : "+str(round(lag,2))+" hours"+'\n'
+					tstamp7 = "Timestamp on Mirror   : "+str(ots)
+					curtime7 = "Current time in UTC   : "+str(time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))+'\n'
+					bugurl7 = "Bug URL : https://bugs.gentoo.org/"+bugnum+" , "+duedate+'\n'
+					report.write(info7)
+					report.write(tstamp7)
+					report.write(curtime7)
+					report.write(bugurl7)
+					report.close()
+					print info7,tstamp7,curtime7,bugurl7
+				os.unlink('timestamp.chk')