Initial version

Signed-off-by: Max Magorsch <arzano@gentoo.org>

21 files changed, 1383 insertions, 0 deletions

diff --git a/pkg/app/home/home.go b/pkg/app/home/home.go
new file mode 100644
index 0000000..e2d3955
--- /dev/null
+++ b/pkg/app/home/home.go
@@ -0,0 +1,71 @@
+// Used to show the landing page of the application
+
+package home
+
+import (
+	"archives/pkg/app/popular"
+	"archives/pkg/config"
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"github.com/go-pg/pg/v10/orm"
+	"net/http"
+	"time"
+)
+
+// Show renders a template to show the landing page of the application
+func Show(w http.ResponseWriter, r *http.Request) {
+
+	var mailingLists []models.MailingList
+
+	for _, mailingList := range config.IndexMailingLists() {
+		var messages []*models.Message
+		database.DBCon.Model(&messages).
+			WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+				q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + mailingList[0] + `]%'`).
+					WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + mailingList[0] + `]%'`)
+				return q, nil
+			}).
+			WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+				q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
+					WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@lists.gentoo.org%'`).
+					WhereOr(`headers::jsonb->>'To' LIKE '%` + mailingList[0] + `@gentoo.org%'`).
+					WhereOr(`headers::jsonb->>'Cc' LIKE '%` + mailingList[0] + `@gentoo.org%'`)
+				return q, nil
+			}).
+			Order("date DESC").
+			Limit(5).
+			Select()
+
+		mailingLists = append(mailingLists, models.MailingList{
+			Name:        mailingList[0],
+			Description: mailingList[1],
+			Messages:    messages,
+		})
+	}
+
+	//
+	// Get popular threads
+	//
+	popularThreads, err := popular.GetPopularThreads(10, "2020-06-01")
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+	if len(popularThreads) > 5 {
+		popularThreads = popularThreads[:5]
+	}
+
+	templateData := struct {
+		MailingLists   []models.MailingList
+		PopularThreads models.Threads
+		MessageCount   string
+		CurrentMonth   string
+	}{
+		MailingLists:   mailingLists,
+		PopularThreads: popularThreads,
+		MessageCount:   formatMessageCount(getAllMessagesCount()),
+		CurrentMonth:   time.Now().Format("2006-01"),
+	}
+
+	renderIndexTemplate(w, templateData)
+}
diff --git a/pkg/app/home/utils.go b/pkg/app/home/utils.go
new file mode 100644
index 0000000..daab2de
--- /dev/null
+++ b/pkg/app/home/utils.go
@@ -0,0 +1,64 @@
+// miscellaneous utility functions used for the landing page of the application
+
+package home
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"github.com/go-pg/pg/v10"
+	"html/template"
+	"net/http"
+	"strconv"
+)
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderIndexTemplate(w http.ResponseWriter, templateData interface{}) {
+	templates := template.Must(
+		template.Must(
+			template.New("Show").
+				Funcs(template.FuncMap{
+					"makeMessage" : func(headers map[string][]string) models.Message {
+						return models.Message{
+							Headers:     headers,
+						}
+					},
+				}).
+				ParseGlob("web/templates/layout/*.tmpl")).
+			ParseGlob("web/templates/home/*.tmpl"))
+
+	templates.ExecuteTemplate(w, "home.tmpl", templateData)
+}
+
+// utility methods
+
+func getAllMessagesCount() int {
+	var messsageCount int
+	database.DBCon.Model((*models.Message)(nil)).QueryOne(pg.Scan(&messsageCount), `
+		SELECT
+			count(DISTINCT messages.headers->>'Message-Id')
+		FROM
+			messages;
+	`)
+	return messsageCount
+}
+
+// formatMessageCount returns the formatted number of
+// messages containing a thousands comma
+func formatMessageCount(messageCount int) string {
+	packages := strconv.Itoa(messageCount)
+	if len(string(messageCount)) == 9 {
+		return packages[:3] + "," + packages[3:6] + "," + packages[6:]
+	} else if len(packages) == 8 {
+		return packages[:2] + "," + packages[2:5] + "," + packages[5:]
+	} else if len(packages) == 7 {
+		return packages[:1] + "," + packages[1:4] + "," + packages[4:]
+	} else if len(packages) == 6 {
+		return packages[:3] + "," + packages[3:]
+	} else if len(packages) == 5 {
+		return packages[:2] + "," + packages[2:]
+	} else if len(packages) == 4 {
+		return packages[:1] + "," + packages[1:]
+	} else {
+		return packages
+	}
+}
diff --git a/pkg/app/list/browse.go b/pkg/app/list/browse.go
new file mode 100644
index 0000000..7d046e6
--- /dev/null
+++ b/pkg/app/list/browse.go
@@ -0,0 +1,40 @@
+package list
+
+import (
+	"archives/pkg/config"
+	"archives/pkg/models"
+	"net/http"
+)
+
+func Browse(w http.ResponseWriter, r *http.Request) {
+
+	// Count number of messages in the current mailing lists
+	var currentMailingLists []models.MailingList
+	for _, listName := range config.CurrentMailingLists() {
+		messageCount, _ := countMessages(listName)
+		currentMailingLists = append(currentMailingLists, models.MailingList{
+			Name:         listName,
+			MessageCount: messageCount,
+		})
+	}
+
+	// Count number of messages in the frozen archives
+	var frozenArchives []models.MailingList
+	for _, listName := range config.FrozenArchives() {
+		messageCount, _ := countMessages(listName)
+		frozenArchives = append(frozenArchives, models.MailingList{
+			Name:         listName,
+			MessageCount: messageCount,
+		})
+	}
+
+	browseData := struct {
+		CurrentMailingLists []models.MailingList
+		FrozenArchives      []models.MailingList
+	}{
+		CurrentMailingLists: currentMailingLists,
+		FrozenArchives:      frozenArchives,
+	}
+
+	renderBrowseTemplate(w, browseData)
+}
diff --git a/pkg/app/list/messages.go b/pkg/app/list/messages.go
new file mode 100644
index 0000000..383e891
--- /dev/null
+++ b/pkg/app/list/messages.go
@@ -0,0 +1,59 @@
+package list
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"github.com/go-pg/pg/v10/orm"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+func Messages(w http.ResponseWriter, r *http.Request) {
+
+	urlParts := strings.Split(r.URL.Path, "/messages/")
+	if len(urlParts) != 2 {
+		http.NotFound(w, r)
+		return
+	}
+
+	listName := strings.ReplaceAll(urlParts[0], "/", "")
+
+	trailingUrlParts := strings.Split(urlParts[1], "/")
+	combinedDate := trailingUrlParts[0]
+	currentPage := 1
+	if len(trailingUrlParts) > 1 {
+		parsedCurrentPage, err := strconv.Atoi(trailingUrlParts[1])
+		if err == nil {
+			currentPage = parsedCurrentPage
+		}
+	}
+	offset := (currentPage - 1) * 50
+
+	var messages []*models.Message
+	query := database.DBCon.Model(&messages).
+		Column("id", "headers", "date").
+		Where("to_char(date, 'YYYY-MM') = ?", combinedDate).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+			return q, nil
+		}).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			return q, nil
+		}).
+		Order("date DESC")
+
+	messagesCount, _ := query.Count()
+	query.Limit(50).Offset(offset).Select()
+
+	maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+
+	renderMessagesTemplate(w, listName, combinedDate, currentPage, maxPages, messages)
+
+}
diff --git a/pkg/app/list/show.go b/pkg/app/list/show.go
new file mode 100644
index 0000000..8db8778
--- /dev/null
+++ b/pkg/app/list/show.go
@@ -0,0 +1,44 @@
+package list
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"github.com/go-pg/pg/v10/orm"
+	"net/http"
+	"strings"
+)
+
+func Show(w http.ResponseWriter, r *http.Request) {
+
+	listName := strings.ReplaceAll(r.URL.Path, "/", "")
+
+	var res []struct {
+		CombinedDate string
+		MessageCount int
+	}
+	err := database.DBCon.Model((*models.Message)(nil)).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+			return q, nil
+		}).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			return q, nil
+		}).
+		ColumnExpr("to_char(date, 'YYYY-MM') AS combined_date").
+		ColumnExpr("count(*) AS message_count").
+		Group("combined_date").
+		Order("combined_date DESC").
+		Select(&res)
+
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	renderShowTemplate(w, listName, res)
+}
diff --git a/pkg/app/list/threads.go b/pkg/app/list/threads.go
new file mode 100644
index 0000000..33ade3c
--- /dev/null
+++ b/pkg/app/list/threads.go
@@ -0,0 +1,60 @@
+package list
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"github.com/go-pg/pg/v10/orm"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+func Threads(w http.ResponseWriter, r *http.Request) {
+
+	urlParts := strings.Split(r.URL.Path, "/threads/")
+	if len(urlParts) != 2 {
+		http.NotFound(w, r)
+		return
+	}
+
+	listName := strings.ReplaceAll(urlParts[0], "/", "")
+	trailingUrlParts := strings.Split(urlParts[1], "/")
+	combinedDate := trailingUrlParts[0]
+	currentPage := 1
+	if len(trailingUrlParts) > 1 {
+		parsedCurrentPage, err := strconv.Atoi(trailingUrlParts[1])
+		if err == nil {
+			currentPage = parsedCurrentPage
+		}
+	}
+	offset := (currentPage - 1) * 50
+
+	var messages []*models.Message
+	query := database.DBCon.Model(&messages).
+		Column("id", "headers", "date").
+		Where("to_char(date, 'YYYY-MM') = ?", combinedDate).
+		Where(`NOT headers::jsonb ? 'References'`).
+		Where(`NOT headers::jsonb ? 'In-Reply-To'`).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+			return q, nil
+		}).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			return q, nil
+		}).
+		Order("date DESC")
+
+	messagesCount, _ := query.Count()
+	query.Limit(50).Offset(offset).Select()
+
+	maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+
+	renderThreadsTemplate(w, listName, combinedDate, currentPage, maxPages, messages)
+
+}
diff --git a/pkg/app/list/utils.go b/pkg/app/list/utils.go
new file mode 100644
index 0000000..07dbc65
--- /dev/null
+++ b/pkg/app/list/utils.go
@@ -0,0 +1,137 @@
+// miscellaneous utility functions used for the landing page of the application
+
+package list
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"github.com/go-pg/pg/v10/orm"
+	"html/template"
+	"net/http"
+)
+
+type ListData struct {
+	ListName    string
+	Date        string
+	CurrentPage int
+	MaxPages    int
+	Messages    []*models.Message
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderShowTemplate(w http.ResponseWriter, listName string, messageData interface{}) {
+	templates := template.Must(
+		template.Must(
+			template.New("Show").
+				ParseGlob("web/templates/layout/*.tmpl")).
+			ParseGlob("web/templates/list/*.tmpl"))
+
+	templteData := struct {
+		ListName    string
+		MessageData interface{}
+	}{
+		ListName:    listName,
+		MessageData: messageData,
+	}
+
+	templates.ExecuteTemplate(w, "show.tmpl", templteData)
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderMessagesTemplate(w http.ResponseWriter, listName string, date string, currentPage int, maxPages int, messages []*models.Message) {
+	templates := template.Must(
+		template.Must(
+			template.Must(
+				template.New("Show").
+					Funcs(getFuncMap()).
+					ParseGlob("web/templates/layout/*.tmpl")).
+				ParseGlob("web/templates/list/components/*.tmpl")).
+			ParseGlob("web/templates/list/*.tmpl"))
+
+	templates.ExecuteTemplate(w, "messages.tmpl", buildListData(listName, date, currentPage, maxPages, messages))
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderThreadsTemplate(w http.ResponseWriter, listName string, date string, currentPage int, maxPages int, messages []*models.Message) {
+	templates := template.Must(
+		template.Must(
+			template.Must(
+				template.New("Show").
+					Funcs(getFuncMap()).
+					ParseGlob("web/templates/layout/*.tmpl")).
+				ParseGlob("web/templates/list/components/*.tmpl")).
+			ParseGlob("web/templates/list/*.tmpl"))
+
+	templates.ExecuteTemplate(w, "threads.tmpl", buildListData(listName, date, currentPage, maxPages, messages))
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderBrowseTemplate(w http.ResponseWriter, lists interface{}) {
+	templates := template.Must(
+		template.Must(
+			template.New("Show").
+				ParseGlob("web/templates/layout/*.tmpl")).
+			ParseGlob("web/templates/list/*.tmpl"))
+
+	templates.ExecuteTemplate(w, "browse.tmpl", lists)
+}
+
+// utility methods
+
+func getFuncMap() template.FuncMap {
+	return template.FuncMap{
+		"min": func(a, b int) int {
+			if a < b {
+				return a
+			}
+			return b
+		},
+		"max": func(a, b int) int {
+			if a < b {
+				return b
+			}
+			return a
+		},
+		"add": func(a, b int) int {
+			return a + b
+		},
+		"sub": func(a, b int) int {
+			return a - b
+		},
+		"makeRange": makeRange,
+	}
+}
+
+func buildListData(listName string, date string, currentPage int, maxPages int, messages []*models.Message) ListData {
+	return ListData{
+		ListName:    listName,
+		Date:        date,
+		CurrentPage: currentPage,
+		MaxPages:    maxPages,
+		Messages:    messages,
+	}
+}
+
+func makeRange(min, max int) []int {
+	a := make([]int, max-min+1)
+	for i := range a {
+		a[i] = min + i
+	}
+	return a
+}
+
+func countMessages(listName string) (int, error) {
+	return database.DBCon.Model((*models.Message)(nil)).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE '[` + listName + `]%'`).
+				WhereOr(`(headers::jsonb->>'Subject')::jsonb->>0 LIKE 'Re: [` + listName + `]%'`)
+			return q, nil
+		}).
+		WhereGroup(func(q *orm.Query) (*orm.Query, error) {
+			q = q.WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@lists.gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'To' LIKE '%` + listName + `@gentoo.org%'`).
+				WhereOr(`headers::jsonb->>'Cc' LIKE '%` + listName + `@gentoo.org%'`)
+			return q, nil
+		}).Count()
+}
diff --git a/pkg/app/message/show.go b/pkg/app/message/show.go
new file mode 100644
index 0000000..a027596
--- /dev/null
+++ b/pkg/app/message/show.go
@@ -0,0 +1,49 @@
+// Used to show the landing page of the application
+
+package message
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"net/http"
+	"strings"
+)
+
+// Show renders a template to show the landing page of the application
+func Show(w http.ResponseWriter, r *http.Request) {
+
+	urlParts := strings.Split(r.URL.Path, "/")
+	listName := urlParts[1]
+	messageHash := urlParts[len(urlParts)-1]
+
+	message := &models.Message{Id: messageHash}
+	err := database.DBCon.Select(message)
+
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	var inReplyTos []*models.Message
+	var inReplyTo *models.Message
+	if message.HasHeaderField("In-Reply-To") {
+		err = database.DBCon.Model(&inReplyTos).
+			Where(`(headers::jsonb->>'Message-Id')::jsonb ? '` + message.GetHeaderField("In-Reply-To") + `'`).
+			Select()
+		if err != nil || len(inReplyTos) < 1 {
+			inReplyTo = nil
+		} else {
+			inReplyTo = inReplyTos[0]
+		}
+	} else {
+		inReplyTo = nil
+	}
+
+	var replies []*models.Message
+	database.DBCon.Model(&replies).
+		Where(`(headers::jsonb->>'References')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`).
+		WhereOr(`(headers::jsonb->>'In-Reply-To')::jsonb ? '` + message.GetHeaderField("Message-Id") + `'`).
+		Order("date ASC").Select()
+
+	renderMessageTemplate(w, listName, message, inReplyTo, replies)
+}
diff --git a/pkg/app/message/utils.go b/pkg/app/message/utils.go
new file mode 100644
index 0000000..0cb40f5
--- /dev/null
+++ b/pkg/app/message/utils.go
@@ -0,0 +1,67 @@
+// miscellaneous utility functions used for the landing page of the application
+
+package message
+
+import (
+	"archives/pkg/models"
+	"html/template"
+	"net/http"
+	"strings"
+)
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderMessageTemplate(w http.ResponseWriter, listName string, message *models.Message, inReplyTo *models.Message, replies []*models.Message) {
+	templates := template.Must(
+		template.Must(
+			template.New("Show").
+				Funcs(getFuncMap()).
+				ParseGlob("web/templates/layout/*.tmpl")).
+			ParseGlob("web/templates/message/*.tmpl"))
+
+	templateData := struct {
+		ListName  string
+		Message   *models.Message
+		InReplyTo *models.Message
+		Replies   []*models.Message
+	}{
+		ListName:  listName,
+		Message:   message,
+		InReplyTo: inReplyTo,
+		Replies:   replies,
+	}
+
+	templates.ExecuteTemplate(w, "show.tmpl", templateData)
+}
+
+func getFuncMap() template.FuncMap {
+	return template.FuncMap{
+		"formatAddr": func(addr string) string {
+			if strings.Contains(addr, "@lists.gentoo.org") || strings.Contains(addr, "@gentoo.org") {
+				addr = strings.ReplaceAll(addr, "@lists.gentoo.org", "@l.g.o")
+				addr = strings.ReplaceAll(addr, "@gentoo.org", "@g.o")
+			} else {
+				start := false
+				for i := len(addr) - 1; i > 0; i-- {
+					if addr[i] == '@' {
+						break
+					}
+					if start {
+						out := []rune(addr)
+						out[i] = '×'
+						addr = string(out)
+					}
+					if addr[i] == '.' {
+						start = true
+					}
+				}
+			}
+			return addr
+		},
+	}
+}
+
+func replaceAtIndex(in string, r rune, i int) string {
+	out := []rune(in)
+	out[i] = r
+	return string(out)
+}
diff --git a/pkg/app/popular/threads.go b/pkg/app/popular/threads.go
new file mode 100644
index 0000000..5c12c8a
--- /dev/null
+++ b/pkg/app/popular/threads.go
@@ -0,0 +1,14 @@
+package popular
+
+import (
+	"net/http"
+)
+
+func Threads(w http.ResponseWriter, r *http.Request) {
+	threads, err := GetPopularThreads(25, "2020-06-01")
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+	renderPopularThreads(w, threads)
+}
diff --git a/pkg/app/popular/utils.go b/pkg/app/popular/utils.go
new file mode 100644
index 0000000..6cfeff7
--- /dev/null
+++ b/pkg/app/popular/utils.go
@@ -0,0 +1,57 @@
+package popular
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"html/template"
+	"net/http"
+	"strconv"
+)
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderPopularThreads(w http.ResponseWriter, templateData interface{}) {
+	templates := template.Must(
+		template.Must(
+			template.New("Popular").
+				Funcs(template.FuncMap{
+					"makeMessage" : func(headers map[string][]string) models.Message {
+						return models.Message{
+							Headers:     headers,
+						}
+					},
+			}).
+				ParseGlob("web/templates/layout/*.tmpl")).
+			ParseGlob("web/templates/popular/*.tmpl"))
+
+	templates.ExecuteTemplate(w, "threads.tmpl", templateData)
+}
+
+// utility methods
+
+func GetPopularThreads(n int, date string) (models.Threads, error) {
+	var popularThreads models.Threads
+	err := database.DBCon.Model(&popularThreads).
+		TableExpr(`(SELECT id, headers, regexp_replace(regexp_replace(regexp_replace(regexp_replace(headers::jsonb->>'Subject','^\["',''),'"\]$',''),'^Re:\s',''), '^\[.*\]', '') AS c FROM messages WHERE date >= '2020-06-12'::date) t`).
+		ColumnExpr(`c as Subject, jsonb_agg(id)->>0 as Id, jsonb_agg(headers)->>0 as Headers, Count(*) as Count`).
+		GroupExpr(`c`).
+		OrderExpr(`count DESC`).
+		Limit(n).
+		Select()
+
+	return popularThreads, err
+}
+
+func GetMessagesFromPopularThreads(threads models.Threads) []*models.Message {
+	var popularThreads []*models.Message
+	for _, thread := range threads {
+		var messages []*models.Message
+		err := database.DBCon.Model(&messages).
+			Where(`headers::jsonb->>'Subject' LIKE '%` + thread.Id + `%'`).
+			Select()
+		if err == nil && len(messages) > 0 {
+			messages[0].Comment = strconv.Itoa(thread.Count)
+			popularThreads = append(popularThreads, messages[0])
+		}
+	}
+	return popularThreads
+}
diff --git a/pkg/app/search/search.go b/pkg/app/search/search.go
new file mode 100644
index 0000000..f6498e9
--- /dev/null
+++ b/pkg/app/search/search.go
@@ -0,0 +1,98 @@
+package search
+
+import (
+	"archives/pkg/config"
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+func Search(w http.ResponseWriter, r *http.Request) {
+
+	//
+	// Parse search params
+	//
+	searchTerm := getParameterValue("q", r)
+	showThreads := getParameterValue("threads", r) != ""
+	page, err := strconv.Atoi(getParameterValue("page", r))
+	var currentPage int
+	var offset int
+
+	if err != nil {
+		currentPage = 1
+		offset = 0
+	} else {
+		currentPage = page
+		offset = 50 * (page - 1)
+	}
+
+	//
+	// Step 1: Search for List with the same name and redirect
+	//
+	for _, list := range config.AllPublicMailingLists() {
+		if strings.TrimSpace(searchTerm) == list {
+			http.Redirect(w, r, "/"+list+"/", http.StatusMovedPermanently)
+			return
+		}
+	}
+
+	//
+	// Step 2: Search by Author
+	//
+	var searchResults []*models.Message
+	query := database.DBCon.Model(&searchResults).
+		WhereOr(`headers::jsonb->>'From' LIKE ?`, "%"+searchTerm+"%").
+		Order("date DESC")
+	if showThreads {
+		query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+	}
+
+	messagesCount, _ := query.Count()
+	err = query.Limit(50).Offset(offset).Select()
+
+	if err == nil && messagesCount > 0 && strings.TrimSpace(searchTerm) != "gentoo" {
+		maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+		renderSearchTemplate(w, showThreads, searchTerm, messagesCount, currentPage, maxPages, searchResults)
+		return
+	}
+
+	//
+	// Step 3: Search by Subject
+	//
+	query = database.DBCon.Model(&searchResults).
+		Where(`tsv_subject @@ to_tsquery(''?'')`, searchTerm)
+	if showThreads {
+		query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+	}
+
+	messagesCount, _ = query.Count()
+	err = query.Limit(50).Offset(offset).Select()
+
+	if err == nil && messagesCount > 0 {
+		maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+		renderSearchTemplate(w, showThreads, searchTerm, messagesCount, currentPage, maxPages, searchResults)
+		return
+	}
+
+	//
+	// Step 4: Search by Message Body
+	//
+	query = database.DBCon.Model(&searchResults).
+		Where(`tsv_body @@ to_tsquery(''?'')`, searchTerm)
+	if showThreads {
+		query = query.Where(`NOT headers::jsonb ? 'References'`).Where(`NOT headers::jsonb ? 'In-Reply-To'`)
+	}
+
+	messagesCount, _ = query.Count()
+	err = query.Limit(50).Offset(offset).Select()
+
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+	maxPages := int(math.Ceil(float64(messagesCount) / float64(50)))
+	renderSearchTemplate(w, showThreads, searchTerm, messagesCount, currentPage, maxPages, searchResults)
+}
diff --git a/pkg/app/search/utils.go b/pkg/app/search/utils.go
new file mode 100644
index 0000000..0fa4285
--- /dev/null
+++ b/pkg/app/search/utils.go
@@ -0,0 +1,90 @@
+package search
+
+import (
+	"archives/pkg/models"
+	"html/template"
+	"net/http"
+)
+
+type SearchData struct {
+	SearchQuery        string
+	ShowThreads        bool
+	SearchResultsCount int
+	CurrentPage        int
+	MaxPages           int
+	Messages           []*models.Message
+}
+
+// renderIndexTemplate renders all templates used for the landing page
+func renderSearchTemplate(w http.ResponseWriter, showThreads bool, searchQuery string, messagesCount int, currentPage int, maxPages int, messages []*models.Message) {
+	templates := template.Must(
+		template.Must(
+			template.Must(
+				template.New("Show").
+					Funcs(getFuncMap()).
+					ParseGlob("web/templates/layout/*.tmpl")).
+				ParseGlob("web/templates/search/components/pagination.tmpl")).
+			ParseGlob("web/templates/search/*.tmpl"))
+
+	templates.ExecuteTemplate(w, "searchresults.tmpl", buildSearchData(showThreads, searchQuery, messagesCount, currentPage, maxPages, messages))
+}
+
+// utility methods
+
+func getFuncMap() template.FuncMap {
+	return template.FuncMap{
+		"min": func(a, b int) int {
+			if a < b {
+				return a
+			}
+			return b
+		},
+		"max": func(a, b int) int {
+			if a < b {
+				return b
+			}
+			return a
+		},
+		"add": func(a, b int) int {
+			return a + b
+		},
+		"sub": func(a, b int) int {
+			return a - b
+		},
+		"mul": func(a, b int) int {
+			return a * b
+		},
+		"makeRange": makeRange,
+	}
+}
+
+func buildSearchData(showThreads bool, searchQuery string, messagesCount int, currentPage int, maxPages int, messages []*models.Message) SearchData {
+	return SearchData{
+		SearchQuery:        searchQuery,
+		ShowThreads:        showThreads,
+		SearchResultsCount: messagesCount,
+		CurrentPage:        currentPage,
+		MaxPages:           maxPages,
+		Messages:           messages,
+	}
+}
+
+func makeRange(min, max int) []int {
+	a := make([]int, max-min+1)
+	for i := range a {
+		a[i] = min + i
+	}
+	return a
+}
+
+// getParameterValue returns the value of a given parameter
+func getParameterValue(parameterName string, r *http.Request) string {
+	results, ok := r.URL.Query()[parameterName]
+	if !ok {
+		return ""
+	}
+	if len(results) == 0 {
+		return ""
+	}
+	return results[0]
+}
diff --git a/pkg/app/serve.go b/pkg/app/serve.go
new file mode 100644
index 0000000..62eac33
--- /dev/null
+++ b/pkg/app/serve.go
@@ -0,0 +1,60 @@
+// Entrypoint for the web application
+
+package app
+
+import (
+	"archives/pkg/app/home"
+	"archives/pkg/app/list"
+	"archives/pkg/app/message"
+	"archives/pkg/app/popular"
+	"archives/pkg/app/search"
+	"archives/pkg/config"
+	"fmt"
+	"log"
+	"net/http"
+)
+
+// Serve is used to serve the web application
+func Serve() {
+
+	fmt.Println("Serving on Port " + config.Port())
+
+	for _, mailingList := range config.AllPublicMailingLists() {
+		setRoute("/"+mailingList+"/message/", message.Show)
+		setRoute("/"+mailingList+"/messages/", list.Messages)
+		setRoute("/"+mailingList+"/threads/", list.Threads)
+		setRoute("/"+mailingList+"/", list.Show)
+	}
+
+	setRoute("/lists", list.Browse)
+
+	setRoute("/popular", popular.Threads)
+
+	setRoute("/search", search.Search)
+
+	setRoute("/", home.Show)
+
+	fs := http.StripPrefix("/assets/", http.FileServer(http.Dir("assets")))
+	http.Handle("/assets/", fs)
+
+	log.Fatal(http.ListenAndServe(":"+config.Port(), nil))
+
+}
+
+// define a route using the default middleware and the given handler
+func setRoute(path string, handler http.HandlerFunc) {
+	http.HandleFunc(path, mw(handler))
+}
+
+// mw is used as default middleware to set the default headers
+func mw(handler http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		setDefaultHeaders(w)
+		handler(w, r)
+	}
+}
+
+// setDefaultHeaders sets the default headers that apply for all pages
+func setDefaultHeaders(w http.ResponseWriter) {
+	w.Header().Set("Cache-Control", config.CacheControl())
+}
diff --git a/pkg/config/config.go b/pkg/config/config.go
new file mode 100644
index 0000000..70ab9cc
--- /dev/null
+++ b/pkg/config/config.go
@@ -0,0 +1,75 @@
+package config
+
+import (
+	"os"
+	"strings"
+)
+
+func MailDirPath() string {
+	mailDir := getEnv("ARCHIVES_MAILDIR_PATH", "/var/archives/.maildir/")
+	if !strings.HasSuffix(mailDir, "/") {
+		mailDir = mailDir + "/"
+	}
+	return mailDir
+}
+
+func Port() string {
+	return getEnv("ARCHIVES_PORT", "5000")
+}
+
+func PostgresUser() string {
+	return getEnv("ARCHIVES_POSTGRES_USER", "admin")
+}
+
+func PostgresPass() string {
+	return getEnv("ARCHIVES_POSTGRES_PASS", "admin")
+}
+
+func PostgresDb() string {
+	return getEnv("ARCHIVES_POSTGRES_DB", "garchives")
+}
+
+func PostgresHost() string {
+	return getEnv("ARCHIVES_POSTGRES_HOST", "localhost")
+}
+
+func PostgresPort() string {
+	return getEnv("ARCHIVES_POSTGRES_PORT", "5432")
+}
+
+func CacheControl() string {
+	return getEnv("ARCHIVES_CACHE_CONTROL", "max-age=300")
+}
+
+func IndexMailingLists() [][]string {
+	return [][]string{
+		{"gentoo-dev", "is the main technical development mailing list of Gentoo"},
+		{"gentoo-project", "contains non-technical discussion and propositions for the Gentoo Council"},
+		{"gentoo-announce", "contains important news for all Gentoo stakeholders"},
+		{"gentoo-user", "is our main support and Gentoo-related talk mailing list"},
+		{"gentoo-commits", " - Lots of commits"},
+		{"gentoo-dev-announce", "conveys important changes to all developers and interested users"}}
+}
+
+func AllPublicMailingLists() []string {
+	var allMailingLists []string
+	allMailingLists = append(allMailingLists, CurrentMailingLists()...)
+	allMailingLists = append(allMailingLists, FrozenArchives()...)
+	return allMailingLists
+}
+
+func CurrentMailingLists() []string {
+	return []string{"gentoo-announce", "gentoo-commits", "gentoo-dev", "gentoo-dev-announce", "gentoo-nfp", "gentoo-project", "gentoo-user"}
+}
+
+func FrozenArchives() []string {
+	return []string{"gentoo-arm", "gentoo-au", "gentoo-council", "gentoo-cygwin", "gentoo-desktop-research"}
+}
+
+func getEnv(key string, fallback string) string {
+	if os.Getenv(key) != "" {
+		return os.Getenv(key)
+	} else {
+		return fallback
+	}
+}
diff --git a/pkg/database/connection.go b/pkg/database/connection.go
new file mode 100644
index 0000000..18fa2e6
--- /dev/null
+++ b/pkg/database/connection.go
@@ -0,0 +1,78 @@
+// Contains utility functions around the database
+
+package database
+
+import (
+	"archives/pkg/config"
+	"archives/pkg/models"
+	"context"
+	"github.com/go-pg/pg/v10"
+	"github.com/go-pg/pg/v10/orm"
+)
+
+// DBCon is the connection handle
+// for the database
+var (
+	DBCon *pg.DB
+)
+
+// CreateSchema creates the tables in the database
+// in case they don't alreay exist
+func CreateSchema() error {
+	if !tableExists("messages") {
+
+		err := DBCon.CreateTable((*models.Message)(nil), &orm.CreateTableOptions{
+			IfNotExists: true,
+		})
+
+		// Add tsvector column for subjects
+		DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_subject tsvector;")
+		DBCon.Exec("CREATE INDEX subject_idx ON messages USING gin(tsv_subject);")
+
+		// Add tsvector column for bodies
+		DBCon.Exec("ALTER TABLE messages ADD COLUMN tsv_body tsvector;")
+		DBCon.Exec("CREATE INDEX body_idx ON messages USING gin(tsv_body);")
+
+		return err
+	}
+	return nil
+}
+
+type dbLogger struct{}
+
+func (d dbLogger) BeforeQuery(c context.Context, q *pg.QueryEvent) (context.Context, error) {
+	return c, nil
+}
+
+// AfterQuery is used to log SQL queries
+func (d dbLogger) AfterQuery(c context.Context, q *pg.QueryEvent) error {
+	// logger.Debug.Println(q.FormattedQuery())
+	return nil
+}
+
+// Connect is used to connect to the database
+// and turn on logging if desired
+func Connect() {
+	DBCon = pg.Connect(&pg.Options{
+		User:     config.PostgresUser(),
+		Password: config.PostgresPass(),
+		Database: config.PostgresDb(),
+		Addr:     config.PostgresHost() + ":" + config.PostgresPort(),
+	})
+
+	DBCon.AddQueryHook(dbLogger{})
+
+	err := CreateSchema()
+	if err != nil {
+		// logger.Error.Println("ERROR: Could not create database schema")
+		// logger.Error.Println(err)
+	}
+
+}
+
+// utility methods
+
+func tableExists(tableName string) bool {
+	_, err := DBCon.Exec("select * from " + tableName + ";")
+	return err == nil
+}
diff --git a/pkg/importer/importer.go b/pkg/importer/importer.go
new file mode 100644
index 0000000..a989238
--- /dev/null
+++ b/pkg/importer/importer.go
@@ -0,0 +1,24 @@
+package importer
+
+import (
+	"archives/pkg/config"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+func FullImport() {
+	err := filepath.Walk(config.MailDirPath(),
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !info.IsDir() && getDepth(path, config.MailDirPath()) >= 1 {
+				importMail(info.Name(), path, config.MailDirPath())
+			}
+			return nil
+		})
+	if err != nil {
+		log.Println(err)
+	}
+}
diff --git a/pkg/importer/utils.go b/pkg/importer/utils.go
new file mode 100644
index 0000000..5672577
--- /dev/null
+++ b/pkg/importer/utils.go
@@ -0,0 +1,125 @@
+package importer
+
+import (
+	"archives/pkg/database"
+	"archives/pkg/models"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime/multipart"
+	"net/mail"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+)
+
+func importMail(name, path, maildirPath string) {
+	file, _ := os.Open(path)
+	m, _ := mail.ReadMessage(file)
+
+	msg := models.Message{
+		Id:          m.Header.Get("X-Archives-Hash"),
+		Filename:    name,
+		Headers:     m.Header,
+		Attachments: nil,
+		Body:        getBody(m.Header, m.Body),
+		Date:        getDate(m.Header),
+		Lists:       getLists(m.Header),
+		Comment:     "",
+		Hidden:      false,
+	}
+
+	err := insertMessage(msg)
+
+	if err != nil {
+		fmt.Println("Error during importing Mail")
+		fmt.Println(err)
+	}
+}
+
+func getDepth(path, maildirPath string) int {
+	return strings.Count(strings.ReplaceAll(path, maildirPath, ""), "/")
+}
+
+func getBody(header mail.Header, body io.Reader) map[string]string {
+	if isMultipartMail(header) {
+		boundary := regexp.MustCompile(`boundary="(.*?)"`).
+			FindStringSubmatch(
+				header.Get("Content-Type"))
+		if len(boundary) != 2 {
+			//err
+			return map[string]string{
+				"text/plain": "",
+			}
+		}
+		return getBodyParts(body, boundary[1])
+	} else {
+		content, _ := ioutil.ReadAll(body)
+		return map[string]string{
+			getContentType(header): string(content),
+		}
+	}
+}
+
+func getBodyParts(body io.Reader, boundary string) map[string]string {
+	bodyParts := make(map[string]string)
+	mr := multipart.NewReader(body, boundary)
+	for {
+		p, err := mr.NextPart()
+		if err != nil {
+			return bodyParts
+		}
+		slurp, err := ioutil.ReadAll(p)
+		if err != nil {
+			log.Fatal(err)
+		}
+		bodyParts[p.Header.Get("Content-Type")] = string(slurp)
+	}
+	return bodyParts
+}
+
+func getContentType(header mail.Header) string {
+	contentTypes := regexp.MustCompile(`(.*?);`).
+		FindStringSubmatch(
+			header.Get("Content-Type"))
+	if len(contentTypes) < 2 {
+		// assume text/plain if we don't find a Content-Type header e.g. for git patches
+		return "text/plain"
+	}
+	return contentTypes[1]
+}
+
+func getDate(header mail.Header) time.Time {
+	date, _ := header.Date()
+	return date
+}
+
+func isMultipartMail(header mail.Header) bool {
+	return strings.Contains(getContentType(header), "multipart")
+}
+
+func getLists(header mail.Header) []string {
+	var lists []string
+	// To
+	adr, _ := mail.ParseAddressList(header.Get("To"))
+	for _, v := range adr {
+		lists = append(lists, v.Address)
+	}
+	// Cc
+	adr, _ = mail.ParseAddressList(header.Get("Cc"))
+	for _, v := range adr {
+		lists = append(lists, v.Address)
+	}
+	return lists
+}
+
+func insertMessage(message models.Message) error {
+	_, err := database.DBCon.Model(&message).
+		Value("tsv_subject", "to_tsvector(?)", message.GetSubject()).
+		Value("tsv_body", "to_tsvector(?)", message.GetBody()).
+		OnConflict("(id) DO NOTHING").
+		Insert()
+	return err
+}
diff --git a/pkg/models/mailinglist.go b/pkg/models/mailinglist.go
new file mode 100644
index 0000000..b090aed
--- /dev/null
+++ b/pkg/models/mailinglist.go
@@ -0,0 +1,8 @@
+package models
+
+type MailingList struct {
+	Name         string
+	Description  string
+	Messages     []*Message
+	MessageCount int
+}
diff --git a/pkg/models/message.go b/pkg/models/message.go
new file mode 100644
index 0000000..17bbb9d
--- /dev/null
+++ b/pkg/models/message.go
@@ -0,0 +1,155 @@
+package models
+
+import (
+	"mime"
+	"net/mail"
+	"strings"
+	"time"
+)
+
+type Message struct {
+	Id       string `pg:",pk"`
+	Filename string
+
+	Headers     map[string][]string
+	Body        map[string]string
+	Attachments []Attachment
+
+	Lists []string
+	Date  time.Time
+
+	//Search           types.ValueAppender // tsvector
+
+	Comment string
+	Hidden  bool
+
+	//ParentId         string
+	//Parent           Message -> pg fk?
+}
+
+type Header struct {
+	Name    string
+	Content string
+}
+
+type Body struct {
+	ContentType string
+	Content     string
+}
+
+type Attachment struct {
+	Filename string
+	Mime     string
+	Content  string
+}
+
+func (m Message) GetSubject() string {
+	return m.GetHeaderField("Subject")
+}
+
+func (m Message) GetListNameFromSubject() string {
+	subject := m.GetSubject()
+	listName := strings.Split(subject, "]")[0]
+	listName = strings.ReplaceAll(listName, "[", "")
+	listName = strings.ReplaceAll(listName, "Re:", "")
+	listName = strings.TrimSpace(listName)
+	return listName
+}
+
+func (m Message) GetAuthorName() string {
+	addr, err := mail.ParseAddress(m.GetHeaderField("From"))
+	if err != nil {
+		return ""
+	}
+	return addr.Name
+}
+
+func (m Message) GetMessageId() string {
+	messageId := m.GetHeaderField("Message-Id")
+	messageId = strings.ReplaceAll(messageId, "<", "")
+	messageId = strings.ReplaceAll(messageId, ">", "")
+	messageId = strings.ReplaceAll(messageId, "\"", "")
+	return messageId
+}
+
+func (m Message) GetInReplyTo() string {
+	inReplyTo := m.GetHeaderField("In-Reply-To")
+	inReplyTo = strings.ReplaceAll(inReplyTo, "<", "")
+	inReplyTo = strings.ReplaceAll(inReplyTo, ">", "")
+	inReplyTo = strings.ReplaceAll(inReplyTo, " ", "")
+	return inReplyTo
+}
+
+func (m Message) GetHeaderField(key string) string {
+	subject, found := m.Headers[key]
+	if !found {
+		return ""
+	}
+	header := strings.Join(subject, " ")
+	if strings.Contains(header, "=?") {
+		dec := new(mime.WordDecoder)
+		decodedHeader, err := dec.DecodeHeader(header)
+		if err != nil {
+			return ""
+		}
+		return decodedHeader
+	}
+	return header
+}
+
+func (m Message) HasHeaderField(key string) bool {
+	_, found := m.Headers[key]
+	return found
+}
+
+func (m Message) GetBody() string {
+	// Get text/plain body
+	for contentType, content := range m.Body {
+		if strings.Contains(contentType, "text/plain") {
+			return content
+		}
+	}
+
+	// If text/plain is not present, fall back to html
+	for contentType, content := range m.Body {
+		if strings.Contains(contentType, "text/html") {
+			return content
+		}
+	}
+
+	// If neither text/plain nor text/html is available return nothing
+	return ""
+}
+
+func (m Message) HasAttachments() bool {
+	for key, _ := range m.Body {
+		if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) {
+			return true
+		}
+	}
+	return false
+}
+
+func (m Message) GetAttachments() []Attachment {
+	var attachments []Attachment
+	for key, content := range m.Body {
+		if !(strings.Contains(key, "text/plain") || strings.Contains(key, "text/plain")) {
+			attachments = append(attachments, Attachment{
+				Filename: getAttachmentFileName(key),
+				Mime:     strings.Split(key, ";")[0],
+				Content:  content,
+			})
+		}
+	}
+	return attachments
+}
+
+// utility methods
+
+func getAttachmentFileName(contentTypeHeader string) string {
+	parts := strings.Split(contentTypeHeader, "name=")
+	if len(parts) < 2 {
+		return "unknown"
+	}
+	return strings.ReplaceAll(parts[1], "\"", "")
+}
diff --git a/pkg/models/thread.go b/pkg/models/thread.go
new file mode 100644
index 0000000..ebeff2a
--- /dev/null
+++ b/pkg/models/thread.go
@@ -0,0 +1,8 @@
+package models
+
+type Threads []struct {
+	Id         string
+	Headers    map[string][]string
+	Subject    string
+	Count int
+}