16 files changed, 1203 insertions, 0 deletions

diff --git a/src/packages/categories.py b/src/packages/categories.py
new file mode 100644
index 0000000000..cf513bddca
--- /dev/null
+++ b/src/packages/categories.py
@@ -0,0 +1,27 @@
+#!/usr/bin/python -OO
+
+import time
+import MySQLdb
+import ebuilddb
+import config
+import gentoo
+
+def get_categories():
+	db = ebuilddb.db_connect()
+	c = db.cursor()
+	query = ('SELECT DISTINCT(category) '
+		'FROM ebuild '
+		'ORDER BY category ')
+	#print query
+	c.execute(query)
+	results = c.fetchall()
+	return results
+
+categories = [x[0] for x in get_categories()]
+for category in categories:
+	#print category
+	print ('. <a class="altlink" href="%spackages/?category=%s">%s</a><br>' % 
+		(config.FEHOME,
+		category,
+		category))
+print '<br>'
diff --git a/src/packages/changelogs.py b/src/packages/changelogs.py
new file mode 100644
index 0000000000..53335c760b
--- /dev/null
+++ b/src/packages/changelogs.py
@@ -0,0 +1,52 @@
+#!/usr/bin/python -OO
+
+import mstring
+import re
+
+
+BUG_REGEX = re.compile(r'#[0-9]+|bug [0-9]+',re.I)
+BUG_URL = 'http://bugs.gentoo.org/show_bug.cgi?id='
+
+def bugs_to_html(s):
+	index = 1
+	while 1:
+		match = BUG_REGEX.search(s,index)
+		if not match:
+			break
+		start = match.start()
+		end = match.end()
+		substring = s[start:end]
+		if substring[0] == '#':  # this of the form "#1234"
+			bugid = substring[1:]
+		else: # this is of the form "bug 1234"
+			bugid = substring[4:]
+		url = '<a href="%s%s">%s</a>' % (BUG_URL,bugid,substring)
+		(s,index) = mstring.replace_sub(s,url,start,end-1)
+	return s
+
+def changelog(filename):
+	try:
+		#print filename
+		fp = open(filename,'r')
+	except IOError:
+		return ""
+
+
+	s = ""
+	# find first line that isn't blank or a comment
+	while True:
+		line = fp.readline()
+		if not line: break
+		#print line
+		if line[0] not in ['#','','\n']:
+			s = s + line
+			break
+	
+	# append next strings until you reach next "*"
+	while True:
+		line = fp.readline()
+		#print repr(line)
+		if not line or line[0] == '*': break
+		else: s= s + line
+	
+	return s
diff --git a/src/packages/config.py b/src/packages/config.py
new file mode 100644
index 0000000000..7b668213ec
--- /dev/null
+++ b/src/packages/config.py
@@ -0,0 +1,24 @@
+"""config.py  general system configuration parameters
+Every fe module should probably import config"""
+
+__version__ = '20031020.1'
+PORTAGE_DIR = '/usr/portage'
+WEBBASE="/"
+FEHOME="http://packages.gentoo.org/"
+ICONS="%simages" % FEHOME
+ARCHLIST=open('/usr/portage/profiles/arch.list','r').read().split()
+MAXPERPAGE=20
+MAX_RECENT_RELEASES=20
+MAX_CATEGORIES=30
+LOCALHOME="/var/www/packages.gentoo.org/htdocs"
+EBUILD_FILES="%s/ebuilds" % LOCALHOME
+INDEX="main.shtml"
+RSS = 'gentoo.rss'
+RSS2 = 'gentoo_simple.rss'
+RSS_WEBMASTER = 'www@gentoo.org'
+RSS_EDITOR = 'marduk@gentoo.org'
+
+HOST = 'localhost'
+DATABASE = 'fresh_ebuilds'
+USER = 
+PASSWD = 
diff --git a/src/packages/create_ebuild.sql b/src/packages/create_ebuild.sql
new file mode 100644
index 0000000000..2feca8087c
--- /dev/null
+++ b/src/packages/create_ebuild.sql
@@ -0,0 +1,10 @@
+CREATE TABLE ebuild
+(
+	category VARCHAR(30),
+	name VARCHAR(45),
+	version VARCHAR(30) NOT NULL,
+	when_found TIMESTAMP,
+	arch VARCHAR(80),
+	changelog MEDIUMTEXT,
+	PRIMARY KEY (category,name,version)
+)
diff --git a/src/packages/create_package.sql b/src/packages/create_package.sql
new file mode 100644
index 0000000000..abf70fa736
--- /dev/null
+++ b/src/packages/create_package.sql
@@ -0,0 +1,9 @@
+CREATE TABLE package
+(
+	category VARCHAR(30),
+	name VARCHAR(40),
+	homepage VARCHAR(100),
+	description VARCHAR(180),
+	license VARCHAR(30),
+	PRIMARY KEY (category,name)
+)
diff --git a/src/packages/daily.py b/src/packages/daily.py
new file mode 100644
index 0000000000..2907147fed
--- /dev/null
+++ b/src/packages/daily.py
@@ -0,0 +1,112 @@
+#!/usr/bin/python -OO
+
+import sys
+import config
+import ebuilddb
+import gentoo
+import os
+import time
+from genarchbar import genarchbar
+
+print 'Status: 200 Ok'
+print 'Content-type: text/html'
+print
+
+def error():
+	print ('<div class="centerpage">\n'
+	'<table class="ebuild">\n'
+	'<tr><td class="fields">Error in request</td></tr><br>\n'
+	'<tr><td class="item"><img src="%s/?category=generic" align="right" alt="">'
+	'<p>An error was encountered processing your request.  Request a '
+	'different page or check the <a href="%s">fresh ebuilds main page</a>.'
+	'</p></td></tr>'
+	'</table>'
+	'</div>') % (config.ICONS,config.FEHOME)
+
+try:
+	uri = os.environ['REQUEST_URI']
+	datestring = uri.split('/daily/')[1]
+except KeyError:
+	datestring =""
+
+params = datestring.split('/')
+today = time.gmtime(time.time())
+try:
+	year,month,day  = [int(i) for i in params[:3]]
+except ValueError:
+	year,month,day = today[:3]
+
+try:
+	arch = params[4]
+except IndexError:
+	arch = ''
+
+if arch not in [''] + config.ARCHLIST:
+	arch = ''
+
+try:
+	branch = params[5]
+except IndexError:
+	branch = ''
+
+if branch not in ('','stable','testing'):
+	branch = ''
+	
+# see if we're cached, if so write the cache and exit
+if today[:3] == (year,month,day):
+	# today is always cached ;-)
+	archnav = os.path.join(config.LOCALHOME,arch,branch,'archnav.html')
+	filename = os.path.join(config.LOCALHOME,arch,branch,'main.shtml')
+	sys.stdout.write(open(archnav,'r').read())
+	sys.stdout.write('<br>\n')
+	sys.stdout.write(open(filename,'r').read())
+	sys.exit(0)
+else:
+	filename = os.path.join(config.LOCALHOME,'daily','cache',
+		'%d%02d%02d-%s-%s.html' % (year,month,day,arch,branch))
+if os.path.exists(filename):
+	sys.stdout.write(open(filename,'r').read())
+	sys.exit(0)
+	
+db = ebuilddb.db_connect()
+c = db.cursor()
+
+extra = ''
+if arch:
+	if branch == 'stable':
+		extra = ' AND ebuild.arch REGEXP "^%s| %s" ' % (arch,arch)
+	elif branch == 'testing':
+		extra = ' AND ebuild.arch REGEXP "^~%s| ~%s" ' % (arch,arch)
+	else:
+		extra = ' AND ebuild.arch LIKE "%%%s%%" ' % arch
+		
+query = ('SELECT ebuild.category,'
+	'ebuild.name,'
+	'version,'
+	'when_found,'
+	'description,'
+	'changelog,'
+	'arch,'
+	'homepage,'
+	'license '
+	'FROM ebuild,package '
+	'WHERE TO_DAYS(when_found) = TO_DAYS("%s-%02d-%02d") '
+	'AND ebuild.name = package.name '
+	'AND ebuild.category = package.category %s'
+	'ORDER BY when_found desc' %
+	(year, month, day,extra))
+	
+c.execute(query)
+results = c.fetchall()
+
+s = genarchbar('%sdaily/%d/%02d/%02d/' % (config.FEHOME,year,month,day),arch,branch) + '<br>\n'
+for result in results:
+	ebuild = gentoo.query_to_dict(result)
+	s = s + gentoo.ebuild_to_html(ebuild) + '<br>\n'
+print s
+
+# cache to file, if not todays date
+if today[:3] != (year,month,day):
+	filename = os.path.join(config.LOCALHOME,'daily','cache',
+		'%d%02d%02d-%s-%s.html' % (year,month,day,arch,branch))
+	open(filename,'w').write(s)	
diff --git a/src/packages/ebuilddb.py b/src/packages/ebuilddb.py
new file mode 100644
index 0000000000..350a28651d
--- /dev/null
+++ b/src/packages/ebuilddb.py
@@ -0,0 +1,141 @@
+#!/usr/bin/python -OO
+
+import config
+import sys
+import os
+import time
+import re
+import changelogs
+import MySQLdb
+import md5
+
+FINDVER = re.compile('-[0-9]')
+
+def db_connect():
+    return MySQLdb.connect(host = config.HOST,
+        user = config.USER,
+        passwd = config.PASSWD,
+        db = config.DATABASE)
+    
+def find_ebuilds():
+    #print "walking..."
+    ebuilds=[]
+    # yeah, i know we can os.path.walk, but this runs faster ;-)
+    pipe = os.popen("find %s -name '*.ebuild'" % config.PORTAGE_DIR)
+    s = pipe.read()
+    pipe.close()
+    return s.split()
+    
+def parse_ebuild(s):
+    """Parse ebuild info based on path name"""
+    parsed = {}
+    s=s.split('/')
+    parsed['category'] = s[-3]
+    package = s[-1].split('.ebuild')[0]
+    pos=FINDVER.search(package).start()
+    parsed['name'] = package[:pos]
+    parsed['version'] = package[pos+1:]
+
+    return parsed
+
+def get_ebuild_record(db,ebinfo):
+    c = db.cursor()
+    query = ('SELECT * FROM ebuild WHERE category="%s" AND name="%s" '   
+        'AND version="%s" LIMIT 1' % (ebinfo['category'], ebinfo['name'],
+        ebinfo['version']))
+    c.execute(query)
+    result = c.fetchone()
+    return result
+
+def create_ebuild_record(db,ebinfo):
+    c = db.cursor()
+    d = db.cursor()
+    # update the package table
+
+    # this is a really ugly dict comprehension
+    escaped = dict([(x,MySQLdb.escape_string(y)) for (x,y) in ebinfo.items()])
+    query="""REPLACE INTO package VALUES ("%(category)s","%(name)s",\
+        "%(homepage)s","%(description)s","%(license)s");""" % escaped
+    c.execute(query)
+
+    # then add particular ebuild
+    query = ('INSERT INTO ebuild VALUES ("%(category)s","%(name)s",'
+        '"%(version)s",%(time)s,"%(archs)s","%(changelog)s","")' 
+        % escaped)
+    d.execute(query)
+    
+def update_ebuild_record(db,ebinfo):
+    c = db.cursor()
+    escaped = dict([(x,MySQLdb.escape_string(y)) for (x,y) in ebinfo.items()])
+    query = ('UPDATE ebuild '
+        'SET when_found="%(time)s",'
+        'arch="%(archs)s",'
+        'changelog="%(changelog)s",'
+        'prevarch="%(prevarch)s" '
+        'WHERE category="%(category)s" '
+        'AND name="%(name)s" '
+        'AND version="%(version)s" ' % escaped)
+    c.execute(query)
+
+def get_extended_info(ebuild):
+    filename = os.path.join(config.PORTAGE_DIR,'metadata/cache',
+        ebuild['category'],'%s-%s' % (ebuild['name'],ebuild['version']))
+    try:
+        lines = open(filename,'r').readlines()
+    except IOError:
+        lines = []
+    lines = [ s.strip() for s in lines ]
+    try:
+        ebuild['archs'] = lines[8]
+    except IndexError:
+        ebuild['archs'] = ''
+    try:
+        ebuild['homepage'] = lines[5]
+    except IndexError:
+        ebuild['homepage'] = 'http://www.gentoo.org/'
+    try:
+        ebuild['license'] = lines[6]
+    except IndexError:
+        ebuild['license'] = ''
+    try:
+        ebuild['description'] = lines[7]
+    except IndexError:
+        ebuild['description'] = ''
+    return ebuild
+
+def get_mtime(s):
+    """Get mtime of file, return in format that MySQL would like"""
+    try:
+        t = os.path.getmtime(s)
+    except:
+        return 'NULL'
+    str = time.strftime("%Y%m%d%H%M%S",time.localtime(t))
+    return str
+
+def main(): 
+    ebuilds = find_ebuilds()
+    db = db_connect()
+    
+    for s in ebuilds:
+        try:
+            fields = parse_ebuild(s)
+        except:
+            continue
+        result = get_ebuild_record(db,fields)
+        fields = get_extended_info(fields)
+        fields['changelog'] = changelogs.changelog('%s/ChangeLog' 
+            % os.path.dirname(s))
+        fields['time'] = get_mtime(s)
+        if not result:
+            create_ebuild_record(db,fields)
+        elif result[4] != fields['archs']:
+            #print 'ebuild archs=',fields['archs']
+            #print 'db archs=',result[4]
+            #print
+            # keywords change, update db
+            fields['prevarch'] = result[4]
+            update_ebuild_record(db,fields)
+
+        
+if __name__ == '__main__':
+    main()
diff --git a/src/packages/genarchbar.py b/src/packages/genarchbar.py
new file mode 100644
index 0000000000..c96445cad7
--- /dev/null
+++ b/src/packages/genarchbar.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python -OO
+
+
+"""genarchbar.py"""
+
+import config
+import os
+
+def genarchbar(base=config.FEHOME,arch='',branch=''):
+	if arch == '':
+		switch = 'selected'
+	else:
+		switch = 'unselected'
+	if switch == 'selected':
+		colspan = len(config.ARCHLIST)
+	else:
+		colspan = len(config.ARCHLIST) - 2
+	s = ('<table class="archnav" border="0" cellpadding="0" cellspacing="0">\n'
+		'<tr><td colspan="%s" class="%s"><a href="%s">all platforms</a></td>\n' 
+		% (colspan,switch,base))
+	if arch != '':
+		if branch == 'stable':
+			switch = 'selected'
+			click = os.path.join("archs",arch)
+		else:
+			switch = 'unselected'
+			click = os.path.join("archs",arch,"stable")
+		s = '''%s<td class="%s"><a href="%s%s/">stable</a></td>''' % (s,switch,base,click)
+		if branch == 'testing':
+			switch = 'selected'
+			click = os.path.join("archs",arch)
+		else:
+			switch = 'unselected'
+			click = os.path.join("archs",arch,"testing")
+		s = '''%s<td class="%s"><a href="%s%s/">testing</a></td></tr>\n''' % (s,switch,base,click)
+	s = '''%s<tr>''' % s
+	percent = 100/len(config.ARCHLIST)
+	for arch2 in config.ARCHLIST:
+		if arch2 == arch:
+			switch = 'selected'
+			click = ''
+		else:
+			switch = 'unselected'
+			click = os.path.join("archs",arch2)
+		s = ('%s<td class="%s" width="%s%%"><a href="%s%s/">%s</a></td>\n' 
+			% (s,switch,percent,base,click,arch2))
+
+	s = '''%s</tr>\n</table>\n\n''' % s
+	return s
+
+if __name__ == '__main__':
+	symlinks = ['altmenu.html','head.html','index.shtml','menu.html']
+	for arch in [''] + config.ARCHLIST:
+		for branch in ('','stable','testing'):
+			outdir = os.path.join(config.LOCALHOME,"archs",arch,branch)
+			os.system('mkdir -p "%s"' % outdir)
+			if arch != '':
+				for l in symlinks:
+					command ='ln -s %s/archs/%s %s/' % (
+					config.LOCALHOME,l,outdir)
+					print command
+					os.system(command)
+			outfile = os.path.join(outdir,"archnav.html")
+			s = genarchbar(config.FEHOME,arch,branch)
+			open(outfile,'w').write(s)
+
diff --git a/src/packages/genrsslist.py b/src/packages/genrsslist.py
new file mode 100644
index 0000000000..8bd05978f4
--- /dev/null
+++ b/src/packages/genrsslist.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python -OO
+
+import config
+FEHOME = config.FEHOME
+
+print """<div class="centerpage"><H3>Fresh Ebuilds RSS Feeds</H3>
+<p>Fresh Ebuilds provides the following RSS feeds.   Note that some
+feed readers may have problems with the embedded HTML feeds.</p>
+</div>
+<br>
+
+<table class="ebuild">
+	<tr><th class="fields">Platform</th><th class="fields">Branch</th><th class="fields">Format</th><th class="fields">URL</th></tr>
+	<tr class="arch">
+		<th class="arch" rowspan="2">All</th>
+		<td class="arch" rowspan="2">N/A</td>
+		<td class="arch">HTML</td>
+		<td class="arch"><a href="%(FEHOME)sgentoo.rss">
+		%(FEHOME)sgentoo.rss</a></td>
+	</tr>
+	<tr class="arch">
+		<td class="arch">plain</td>
+		<td class="arch"><a href="%(FEHOME)sgentoo_simple.rss">
+		%(FEHOME)sgentoo_simple.rss</a></td>
+	</tr>
+""" % vars()
+
+for arch in config.ARCHLIST:
+	print """
+	<!-- %(arch)s -->
+		<tr class="arch">
+			<th class="arch" rowspan="6">%(arch)s</th>
+			<td class="arch" rowspan="2">All</td>
+			<td class="arch">HTML</td>
+			<td class="arch"><a href="%(FEHOME)sarchs/%(arch)s/gentoo.rss">
+			%(FEHOME)sarchs/%(arch)s/gentoo.rss</a></td>
+		</tr>
+		<tr class="arch">
+			<td class="arch">plain</td>
+			<td class="arch"><a href="%(FEHOME)sarchs/%(arch)s/gentoo_simple.rss">
+			%(FEHOME)sarchs/%(arch)s/gentoo_simple.rss</a></td>
+		</tr>
+		<tr class="arch">
+			<td class="arch" rowspan="2">stable</td>
+			<td class="arch">HTML</td>
+			<td class="arch"><a href="%(FEHOME)sarchs/%(arch)s/stable/gentoo.rss">
+			%(FEHOME)sarchs/%(arch)s/stable/gentoo.rss</a></td>
+		</tr>
+		<tr class="arch">
+			<td class="arch">plain</td>
+			<td class="arch"><a href="%(FEHOME)sarchs/%(arch)s/stable/gentoo_simple.rss">
+			%(FEHOME)sarchs/%(arch)s/stable/gentoo_simple.rss</a></td>
+		</tr>
+			<tr class="arch">
+			<td class="arch" rowspan="2">testing</td>
+			<td class="arch">HTML</td>
+			<td class="arch"><a href="%(FEHOME)sarchs/%(arch)s/testing/gentoo.rss">
+			%(FEHOME)sarchs/%(arch)s/testing/gentoo.rss</a></td>
+		</tr>
+		<tr class="arch">
+			<td class="arch">plain</td>
+			<td class="arch"><a href="%(FEHOME)sarchs/%(arch)s/testing/gentoo_simple.rss">
+			%(FEHOME)sarchs/%(arch)s/testing/gentoo_simple.rss</a></td>
+		</tr>
+	""" % vars()	
+
+print "</table>"
diff --git a/src/packages/gentoo.py b/src/packages/gentoo.py
new file mode 100644
index 0000000000..8279eafa17
--- /dev/null
+++ b/src/packages/gentoo.py
@@ -0,0 +1,329 @@
+#!/usr/bin/python -OO
+
+import config
+import os
+import time
+import sys
+import MySQLdb
+import ebuilddb
+import changelogs
+from cgi import escape
+from urllib import quote
+
+def is_new(db,ebuild):
+    """Check for newness. An ebuild is considered new if it is the
+    only ebuild with that category/name in ebuild and it has no prevarch"""
+    
+    c = db.cursor()
+    query = """SELECT prevarch FROM ebuild WHERE category="%s" AND name="%s" """ \
+        % (ebuild['category'], ebuild['name'])
+    c.execute(query)
+    results = c.fetchall()
+    if len(results) == 1 and results[0][0] is None:
+        return 1
+    return 0
+
+def changelog_to_html(s):
+    n = ""
+    for c in s:
+        if c == '\n':
+            n = "%s<br>" % n
+        else:
+            n = "%s%s" % (n,escape(c))
+    n = changelogs.bugs_to_html(n)
+    return n
+
+def homepage_to_html(s):
+    if not s.strip(): return "?"
+    t = s.replace('|',' ')
+    if ' ' in t:
+        pieces = t.split()
+        return ' '.join([ homepage_to_html(piece) for piece in pieces ])
+    return ("""<a href="%s"><img border="0" src="%s/home.png" width="24" height="20" alt="Home Page" title="Home Page"></a>""" % (s, config.ICONS))
+
+def license_to_html(s):
+    if not s.strip(): return "?"
+    t = s.replace('|',' ')
+    if ' ' in t:
+        pieces = t.split()
+        return "<br>".join([ license_to_html(piece) for piece in pieces ])
+    return """<a href="http://www.gentoo.org/cgi-bin/viewcvs.cgi/*checkout*/licenses/%s">%s</a>""" % (s,s)
+
+def package_to_html(pkginfo,db):
+    """This function needs a database (db) connection because it performs a query_to_dict
+    on the package"""
+    #colspan = len(config.ARCHLIST) + 1
+    s = """<table class="ebuild">
+        <tr><td class="fields" colspan="4">%s</td></tr>
+        <tr><td class="item" colspan="4"><p><img align="right" alt="" src="%s/%s.png">
+        <b>Description: </b>%s</p>
+    """  % (pkginfo['name'],config.ICONS,pkginfo['category'],escape(pkginfo['description']))
+    ebuilds = get_recent_releases(pkginfo,db)
+    if ebuilds:
+        s = """%s<p><b>Releases:</b></p>
+        <table class="releases"><tr><td>&nbsp;</td>""" % s
+        for i in config.ARCHLIST:
+            s = """%s<th class="arch">%s</th>""" % (s,i)
+        s = "%s</tr>\n" % s
+        for ebuild in ebuilds:
+            archs = ebuild['arch'].split()
+            s = """%s<tr><th class="releases"><a href="%sebuilds/?%s-%s" title="%s">%s</a></th>
+            """ % (s,config.FEHOME,ebuild['name'],ebuild['version'],ebuild['time'],ebuild['version'])
+            for arch in config.ARCHLIST:
+                if arch in config.ARCHLIST:
+                    if arch in archs:
+                        arch_string = '+'
+                        title = 'marked stable for %s' % arch
+                    elif '~%s' % arch in archs:
+                        arch_string = '~'
+                        title = 'undergoing testing for %s' % arch
+                    else:
+                        arch_string = '-'
+                        title = 'not available for %s' % arch
+                s = """%s<td class="archcell" arch="%s" title="%s">%s</td>
+                """ % (s,arch_string,title,arch_string)
+            s = "%s</tr>" % s
+        s = "%s</table>\n" % s
+    
+    s = "%s\n%s</table>" % (s,general_info_to_html(pkginfo))
+    return s    
+
+def archs_to_html(archs):
+    arch_string= '|'
+    for arch in config.ARCHLIST:
+        if arch in archs:
+            arch_icon = 'check.gif'
+            alt = ": yes"
+            title = 'marked stable for %s' % arch
+        elif '~%s' % arch in archs:
+            arch_icon = 'bug.gif'
+            alt = ': testing'
+            title = 'undergoing testing for %s' % arch
+        else:
+            arch_icon = 'no.gif'
+            alt = ': no'
+            title = 'not available for %s' % arch
+            
+        arch_string = (' %s %s <img src="%simages/archs/%s" '
+            'alt="%s" title="%s"> |' % (
+            arch_string,
+            arch,
+            config.FEHOME, 
+            arch_icon,
+            alt,
+            title))
+            
+    return arch_string
+                
+def ebuild_to_html(ebinfo,new=0):
+    try:
+        archs = ebinfo['arch'].split()
+    except KeyError:
+        archs = ()
+    arch_string = archs_to_html(archs)
+    changelogurl = ('http://www.gentoo.org/cgi-bin/viewcvs.cgi/*checkout*/'
+        '%s/%s/ChangeLog' % (ebinfo['category'],ebinfo['name']))
+
+    if new:
+        new_string = """ &nbsp;&nbsp;<span class="new">new!</span> """
+    else:
+        new_string = ""
+        
+    return ('<table class="ebuild">\n'
+        '<tr><td colspan="4" class="fields">'
+        '<a href="%spackages/?category=%s;name=%s">%s</a> %s%s<br>'
+        '<span class="time">%s</span><br>\n'
+        '</td></tr>\n'
+        '<tr><td class="item" align="center" colspan="1">'
+        '<img alt="" src="%simages/%s.png">' 
+        '<td class="item" valign="top" colspan="3">'
+        '<p><b>Description:</b> %s</p>\n'
+        '<p><b>Changes:</b><br>\n'
+        '%s</p></td></tr>\n'
+        '<tr><td colspan="4" class="item"><p class="archs">%s</p>\n'
+        '</td></tr>\n'
+        '%s</table>\n'
+        '<br>\n' % (config.FEHOME,
+            quote(ebinfo['category']),
+            quote(ebinfo['name']),
+            ebinfo['name'],
+            ebinfo['version'],
+            new_string,
+            ebinfo['time'].localtime().strftime("%c %Z"),
+            config.FEHOME,
+            ebinfo['category'],
+            escape(ebinfo['description']),
+            changelog_to_html(ebinfo['changelog']),
+            arch_string,
+            general_info_to_html(ebinfo)))
+
+def general_info_to_html(pkg):
+    """This actually will (should) take either a package or ebuild dict
+    as an argument"""
+    
+    changelogurl = ('http://www.gentoo.org/cgi-bin/viewcvs.cgi/*checkout*/'
+        '%s/%s/ChangeLog' % (pkg['category'],pkg['name']))
+    return ('<tr><th class="category">Category</th>'
+        '<td class="homepage" rowspan="2">%s</td>'
+        #'<td class="homepage" rowspan="2"><a href="%s">Home Page</a></td>'
+        '<th class="license">License</th>'
+        '<td class="changelog" rowspan="2"><a href="%s">ChangeLog</a><td>\n'
+        '<tr><td class="category"><a href="%spackages/?category=%s">'
+        '%s</a></td>'
+        #'<td class="homepage"><a href="%s">Home Page</a></td>'
+        '<td class="license">%s</td></tr>\n'
+        % ( homepage_to_html(pkg['homepage']),
+            changelogurl,
+            config.FEHOME,
+            pkg['category'],
+            pkg['category'],
+            #pkg['homepage'],
+            license_to_html(pkg['license'])))
+        
+def get_most_recent(db,max=config.MAXPERPAGE,arch="",branch=""):
+    c = db.cursor()
+    extra = ''
+    if arch:
+        stable_extra = ('ebuild.arch REGEXP "^%s| %s" '
+            ' AND ebuild.prevarch NOT REGEXP"^%s| %s"'
+            % (arch,arch,arch,arch))
+        testing_extra = ('ebuild.arch REGEXP "^~%s| ~%s" '
+            ' AND ebuild.prevarch NOT REGEXP "^~%s| ~%s"'
+            % (arch,arch,arch,arch))
+        if branch == 'stable':
+            extra = ' AND (%s) ' % stable_extra
+        elif branch == 'testing':
+            extra = ' AND (%s) ' % testing_extra
+        else:
+            extra = ' AND ((%s) OR (%s)) ' % (stable_extra, testing_extra)
+    query = """SELECT ebuild.category,ebuild.name,version,when_found,description,
+    changelog,arch,homepage,license FROM ebuild,package WHERE ebuild.name=\
+    package.name AND ebuild.category=package.category %s ORDER by when_found DESC \
+    LIMIT %s""" % (extra,max)
+    c.execute(query)
+    results = c.fetchall()
+    return results
+    
+def query_to_dict(d):
+    einfo = {}
+    keys = ('category','name','version','time','description','changelog',
+        'arch','homepage','license')
+    for i in range(len(keys)):
+        try:
+            einfo[keys[i]] = d[i]
+        except IndexError:
+            continue
+    return einfo
+
+def get_recent_releases(pkg,db,max=config.MAX_RECENT_RELEASES):
+    """Return MAX_RECENT_RELEASES most recent releases for pkg.  Returns and
+    ebuild-type dict"""
+    c = db.cursor()
+    query = """SELECT category,name,version,when_found,NULL,changelog,arch ,NULL,NULL
+    FROM ebuild WHERE name="%s" AND 
+    category="%s" ORDER BY when_found DESC LIMIT %s
+    """ % (pkg['name'],pkg['category'],max)
+    c.execute(query)
+    results = c.fetchall()
+    #print results
+    return [ query_to_dict(i) for i in results ]
+    
+def ebuilds_to_rss(fp,ebuilds,simple=False,subtitle=""):
+    """write out ebuild info to RSS file (fp)"""
+    fp.write("""<?xml version="1.0" encoding="utf-8"?>
+        <rss version="0.92">
+        <channel>
+        <title>Fresh ebuilds %s</title>
+        <link>%s</link>
+        <description>Latest ebuilds from the Gentoo Linux portage tree</description>
+        <webMaster>www@gentoo.org</webMaster>
+        <managingEditor>marduk@gentoo.org</managingEditor>
+        <pubDate>%s</pubDate>""" % (subtitle,config.FEHOME,
+            time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())))
+
+    for ebuild in ebuilds:
+        if simple:
+            description = escape(ebuild['description'])
+        else:
+            description = '\n<![CDATA[\n%s\n]]>' % ebuild_to_html(ebuild)
+
+        fp.write("""<item>
+            <title>%s %s</title>
+            <link>%sebuilds/?%s-%s</link>
+            <description>
+                %s
+            </description>
+            <pubDate>%s</pubDate>
+        </item>
+        """ % (ebuild['name'],
+                ebuild['version'],
+                config.FEHOME,
+                ebuild['name'],
+                ebuild['version'],
+                description,
+                ebuild['time'].gmtime().strftime("%a, %d %b %Y %H:%M:%S +0000"))
+        )
+        
+    fp.write("</channel>\n</rss>\n")
+    
+if __name__ == '__main__':
+    try:
+        if sys.argv[1] == '-g':
+            ebuilddb.main()
+    except IndexError:
+            pass
+
+    db = ebuilddb.db_connect()
+    branches = ('','stable','testing')
+    for arch in [''] + config.ARCHLIST:
+        for branch in branches:
+            fullpath = os.path.join(config.LOCALHOME,"archs",arch,branch,config.INDEX)
+            index = open(fullpath,'w')
+            if arch:
+                sarch = arch
+            else:
+                sarch = 'all'
+            if branch:
+                sbranch = branch
+            else:
+                sbranch = 'all'
+            
+            index.write("""<table border="0" cellpadding="0" cellspacing="5" 
+                width="100%">\n""")
+            index.write("""<tr><td valign="top">\n""")
+            index.write('<!--#include file="archnav.html" -->\n\n</td></tr>\n<tr><td>') 
+            results = get_most_recent(db,arch=arch,branch=branch)
+            ebuilds = [ query_to_dict(i) for i in results ]
+            for ebuild in ebuilds:
+                new = is_new(db,ebuild)
+                pkgfilename = "%s/%s-%s.html" % (
+                    config.EBUILD_FILES,ebuild['name'],ebuild['version'])
+                ebuild_html = ebuild_to_html(ebuild,new)
+                if arch == '' and branch == '':  # and not os.path.exists(pkgfilename):
+                    pkgfile = open(pkgfilename,'w')
+                    pkgfile.write(ebuild_html)
+                    pkgfile.close()
+                    ebuildfilename = "%s/%s/%s/%s-%s.ebuild" % (ebuilddb.config.PORTAGE_DIR,
+                        ebuild['category'],ebuild['name'],ebuild['name'],
+                        ebuild['version'])
+                    os.system('touch -r %s %s || touch -d "today -1 year" %s' % (ebuildfilename,pkgfilename,pkgfilename))
+        
+                try:
+                    index.write('%s<br>\n\n\n' % (ebuild_html))
+                except IOError:
+                    pass
+                #index.write('<!--#include virtual="%s/ebuilds/%s-%s.html" --><br>\n\n\n' 
+                #   % (config.WEBBASE,ebuild['name'],ebuild['version']))
+                #index.write("""<hr class="seperator">\n""")
+                
+            index.write("""</table>\n""")
+            index.close()
+            
+            subtitle = ' %s %s' % (arch,branch)
+            rss = open(os.path.join(config.LOCALHOME,"archs",arch,branch,config.RSS),'w')
+            ebuilds_to_rss(rss,ebuilds,simple=False,subtitle=subtitle)
+            rss.close()
+        
+            rss2 = open(os.path.join(config.LOCALHOME,"archs",arch,branch,config.RSS2),'w')
+            ebuilds_to_rss(rss2,ebuilds,simple=True,subtitle=subtitle)
+            rss.close()
diff --git a/src/packages/mstring.py b/src/packages/mstring.py
new file mode 100644
index 0000000000..59d5d7f361
--- /dev/null
+++ b/src/packages/mstring.py
@@ -0,0 +1,10 @@
+"""marduk's helpin' heap 'o string functions"""
+
+
+def replace_sub(s,new,start,end=None):
+	if end is None:
+		end = start
+	s2 = s[:start] + new + s[end + 1:]
+	newpos = start +  len(new)
+	
+	return (s2,newpos)
diff --git a/src/packages/purge_db.py b/src/packages/purge_db.py
new file mode 100644
index 0000000000..7bf3cc4843
--- /dev/null
+++ b/src/packages/purge_db.py
@@ -0,0 +1,75 @@
+#!/usr/bin/python -OO
+
+import os
+import config
+import ebuilddb
+
+def delete_ebuild(db,ebuild):
+    category, name, version = ebuild[:3]
+    c = db.cursor()
+    query = ('DELETE FROM ebuild WHERE category="%s" AND name="%s" '
+        'AND version="%s"' % (category,name,version))
+    #print query
+    c.execute(query)
+
+def delete_package(db,package):
+    category, name = package[:2]
+    c = db.cursor()
+    query = ('DELETE FROM package WHERE name="%s" AND category="%s"' %
+        (category,name))
+    #print query
+    c.execute(query)
+
+def get_path(ebuild):
+    category, name, version = ebuild[:3]
+    ebuild_fname = os.path.join(config.PORTAGE_DIR,category,name,
+        '%s-%s.ebuild' % (name,version))
+    return ebuild_fname
+
+def ebuilds_in_package(db,package):
+    """using db database, return ebuilds in package"""
+    category, name = package[:2]
+    c = db.cursor()
+    query =('SELECT * FROM ebuild WHERE category="%s" AND NAME="%s"'
+        % (category,name))
+    c.execute(query)
+    results = c.fetchall()
+    return results
+
+def purge_ebuilds(db):
+    """Purge the ebuild table"""
+    c = db.cursor()
+    query = ('SELECT * FROM ebuild')
+    c.execute(query)
+    
+    while 1:
+        ebuild = c.fetchone()
+        if not ebuild: break
+        path = get_path(ebuild)
+        
+        if not os.path.exists(path):
+            print path,' does not exist.'
+            delete_ebuild(db,ebuild)
+
+
+def purge_packages(db):
+    """Purge the package table"""
+    c = db.cursor()
+    query = ('SELECT * FROM package')
+    c.execute(query)
+    
+    while 1:
+        package = c.fetchone()
+        if not package: break
+        count = len(ebuilds_in_package(db,package))
+        if not count:
+            delete_package(db,package)
+
+def main():
+    db = ebuilddb.db_connect()
+    purge_ebuilds(db)
+    purge_packages(db)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/packages/query_ebuild.py b/src/packages/query_ebuild.py
new file mode 100644
index 0000000000..769ff5c858
--- /dev/null
+++ b/src/packages/query_ebuild.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+import cgi
+import os
+import sys
+import config
+import gentoo,ebuilddb
+
+DEFAULT = "404"
+PKG_DIR = config.EBUILD_FILES
+
+if len(sys.argv):
+	ebuild = sys.argv[1]
+else:
+	ebuild = DEFAULT
+
+html_file = os.path.join(PKG_DIR,"%s.html" % ebuild.replace('..',''))
+if os.path.exists(html_file):
+	send_file = html_file
+else:
+	# let's try the database
+	# connect
+	pos=ebuilddb.FINDVER.search(ebuild).start()
+	name = ebuild[:pos]
+	version = ebuild[pos+1:]
+	db = ebuilddb.db_connect()
+	# query
+	query = ('SELECT ebuild.category,ebuild.name,version,when_found,'
+		'description,changelog,arch,homepage,license '
+		'FROM ebuild,package WHERE ebuild.name="%s" AND '
+		'version="%s" AND '
+		'ebuild.name=package.name AND ebuild.category=package.category '
+		'ORDER by when_found DESC LIMIT 1' % (name,version))
+	#print query
+	c = db.cursor()
+	c.execute(query)
+	result = c.fetchone()
+	if result:
+		#print result
+		eb = gentoo.query_to_dict(result)
+		sys.stdout.write(gentoo.ebuild_to_html(eb))
+		sys.exit(0)
+	# else 404
+	else:
+		send_file = os.path.join(PKG_DIR,"%s.html" % DEFAULT)
+
+sys.stdout.write(open(send_file,"r").read())
+sys.stdout.flush()
diff --git a/src/packages/query_package.py b/src/packages/query_package.py
new file mode 100644
index 0000000000..56ec7166be
--- /dev/null
+++ b/src/packages/query_package.py
@@ -0,0 +1,85 @@
+#!/usr/bin/python -OO
+
+import cgi
+from urllib import quote
+import os
+import sys
+import config
+import gentoo
+import ebuilddb
+from MySQLdb import escape_string
+
+sys.stdout.write('Content-type: text/html; charset=UTF-8\n\n')
+
+def query_to_dict(q):
+	pkginfo = {}
+	keys = ('category','name','homepage','description','license')
+	for i in range(len(keys)):
+		try:
+			pkginfo[keys[i]] = q[i]
+		except IndexError:
+			continue
+	return pkginfo
+
+
+form = cgi.FieldStorage()
+name = form.getvalue("name","")
+category = form.getvalue("category","")
+offset = form.getvalue("offset","0")
+
+query = ('SELECT category,name,homepage,description,license '
+	'FROM package WHERE category="%s"' % escape_string(category))
+	
+if name:
+	query = ('%s AND name="%s"' %(query,escape_string(name)))
+
+query = ('%s LIMIT %s,%s' % (query,offset,config.MAX_CATEGORIES))
+
+db = ebuilddb.db_connect()
+c = db.cursor()
+c.execute(query)
+results = c.fetchall()
+
+#print query
+if results:
+	if name:
+		for result in results:
+			#print result
+			pkg = query_to_dict(result)
+			sys.stdout.write('%s<br>\n<br>\n' 
+				% gentoo.package_to_html(pkg,db))
+	else:
+		sys.stdout.write('<table class="centerpage">\n')
+		sys.stdout.write('<tr><td class="fields">'
+			'%s</td><td align="center"><img alt="" src="'
+			'%s/%s.png"></td></tr>\n' 
+			% (category,config.ICONS,category))
+		for result in results:
+			pkg = query_to_dict(result)
+			sys.stdout.write('<tr><td width="25%%" class="fields">'
+				'<a href="?category=%s;name=%s">%s</a></td>'
+				'<td class="description">%s</td></tr>\n'
+				% (pkg['category'],quote(pkg['name']),pkg['name'],
+					cgi.escape(pkg['description'])))
+		sys.stdout.write('</table>\n')
+	if offset !="0":
+		sys.stdout.write('<a href="?category=%s;name=%s'
+			';offset=%s">[Previous]</a> '
+			% (category,name,int(offset) - config.MAX_CATEGORIES))
+	if len(results) == config.MAX_CATEGORIES:
+		sys.stdout.write('<a href="?category=%s;name=%s;offset=%s">[Next]</a> '
+		% (category,name,int(offset) + config.MAX_CATEGORIES))
+
+else:
+	sys.stdout.write('<div class="centerpage">\n'
+		'<table width="100%%" border="0"  align="center"'
+		' cellspacing="0"><tr><td colspan="3" class="fields">'
+		'Sorry, dude.  I could not find that package.<br></td></tr>\n'
+		'<tr><td class="item" colspan="3">'
+		'<img '
+		'src="%s?category=404"'
+		'align="right" alt=""> <p>Information on the package you requested'
+		' could not be found.  Be sure to check the'
+		' <a HREF="%s">'
+		'fresh ebuilds main page</a>.</p></td></tr></table>\n'
+		'</div>' % (config.ICONS,config.FEHOME))
diff --git a/src/packages/search.py b/src/packages/search.py
new file mode 100644
index 0000000000..573990781b
--- /dev/null
+++ b/src/packages/search.py
@@ -0,0 +1,90 @@
+#!/usr/bin/python -OO
+
+import sys
+import config
+import cgi
+import ebuilddb
+import gentoo
+import os
+from MySQLdb import escape_string
+
+form = cgi.FieldStorage()
+type = form.getvalue("type","")
+sstring = form.getvalue("sstring","")
+offset = form.getvalue("offset","0")
+
+if not sstring:
+	sys.exit(0)
+
+if type == 'desc':
+	col = "description"
+else:
+	col = "name"
+
+def query_to_dict(q):
+	pkginfo = {}
+	keys = ('category','name','homepage','description','license')
+	for i in range(len(keys)):
+		try:
+			pkginfo[keys[i]] = q[i]
+		except IndexError:
+			continue
+	return pkginfo
+
+def write_to_cache(s):
+	open(cachefile,'w').write(s)
+
+# if it's in the cache, just write that out
+qs = sys.argv[1]
+cachefile = os.path.join(config.LOCALHOME,'search/cache',qs)
+if os.path.exists(cachefile):
+	#sys.stdout.write('<div class="centerpage">using cached results</div>\n')
+	sys.stdout.write(open(cachefile,'r').read())
+	sys.exit(0)
+
+escaped = escape_string(sstring)
+query = ('SELECT category,name,homepage,description,license '
+	'FROM package WHERE name REGEXP "%s" or description REGEXP "%s" '
+	'LIMIT %s,%s' 
+	% (escaped,escaped,offset,config.MAXPERPAGE))
+
+db = ebuilddb.db_connect()
+c = db.cursor()
+try:
+	c.execute(query)
+	results = c.fetchall()
+except:
+	results = None
+
+if not results:
+	s = ( '<div class="centerpage">\n'
+	'<table width="100%%" border="0"  align="center"'
+	' cellspacing="0"><tr><td colspan="3" class="fields">'
+	'Nothing found.<br></td></tr>\n'
+	'<tr><td class="item" colspan="3">'
+	'<img '
+	'src="%s/?category=generic"'
+	'align="right" alt=""> <p>I could not find any ebuild that match'
+	' your query.  Try a different query or check the'
+	' <a HREF="%s">'
+	'fresh ebuilds main page</a>.</p></td></tr></table>\n'
+	'</div>\n' % (config.ICONS,config.FEHOME))
+	sys.stdout.write(s)
+	write_to_cache(s)
+	sys.exit(0)
+
+pkgs = [ query_to_dict(i) for i in results ]
+
+s = ''
+for pkg in pkgs:
+	#print pkg
+	html = gentoo.package_to_html(pkg,db)
+	s = '%s\n%s<br>\n<br>\n' % (s,gentoo.package_to_html(pkg,db))
+
+if offset != "0":
+	s = '%s<a href="?sstring=%s;offset=%s">[Previous]</a> ' % (s,sstring,int(offset) - config.MAXPERPAGE)
+if len(results) == config.MAXPERPAGE:
+	s = '%s<a href="?sstring=%s;offset=%s">[Next]</a> ' % (s,sstring,int(offset) + config.MAXPERPAGE)
+
+sys.stdout.write(s)
+write_to_cache(s)
diff --git a/src/packages/weeks.py b/src/packages/weeks.py
new file mode 100644
index 0000000000..535a49e964
--- /dev/null
+++ b/src/packages/weeks.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python -OO
+
+import time
+import MySQLdb
+from cgi import escape
+import ebuilddb
+import config
+import gentoo
+
+SECS_PER_DAY = 86400
+DAYS = ('Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday')
+
+def get_dayname(day):
+	return DAYS[day[6]]
+
+def get_days_ebuilds(day):
+	c = db.cursor()
+	query = ('SELECT ebuild.category, '
+		'ebuild.name, '
+		'version, '
+		'when_found, '
+		'description, '
+		'changelog, '
+		'arch, '
+		'homepage, '
+		'license '
+		'FROM ebuild,package '
+		'WHERE TO_DAYS(when_found) = TO_DAYS("%s-%02d-%02d") '
+		'AND ebuild.name = package.name '
+		'AND ebuild.category = package.category '
+		'ORDER BY when_found DESC' %
+		(day[0],day[1],day[2]))
+	#print query
+	c.execute(query)
+	results = c.fetchall()
+	return results
+
+today = time.time()
+db = ebuilddb.db_connect()
+for day in range(today,today - (7*SECS_PER_DAY),-SECS_PER_DAY):
+	#print day
+	gmtime = time.gmtime(day)
+	dayname = get_dayname(gmtime)
+	print ('<a href="%sdaily/%s/%02d/%02d/">%s</a>:<br>' 
+		% (config.FEHOME,gmtime[0],gmtime[1],gmtime[2],dayname))
+	results = get_days_ebuilds(gmtime)
+	#print results
+	ebuilds = [ gentoo.query_to_dict(i) for i in results ]
+	#ebuilds.sort(ebuild_sort)
+	for ebuild in ebuilds:
+		print ('. <a class="altlink" title="%s" href="%sebuilds/?%s-%s">%s %s</a><br>' % 
+		(escape(ebuild['description']),
+		config.FEHOME,
+		ebuild['name'], 
+		ebuild['version'],
+		ebuild['name'],
+		ebuild['version']))
+	print '<br>'