Merge branch 'dev/main'v9.0.1

Signed-off-by: Max Magorsch <arzano@gentoo.org>

11 files changed, 106 insertions, 82 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..681d8e7
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,37 @@
+stages:
+  - build
+  - test
+  - dast
+
+info:
+  stage: .pre
+  script:
+    - docker info
+  
+build:
+  stage: build
+  variables:
+    IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_BRANCH:$CI_COMMIT_SHA
+    LATEST_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_BRANCH:latest
+  script:
+    - echo $IMAGE_TAG
+    - echo $LATEST_IMAGE_TAG
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
+    - docker pull gentoo/portage:latest
+    - docker pull gentoo/stage3-amd64
+    - docker pull $LATEST_IMAGE_TAG || true
+    - docker build --cache-from $LATEST_IMAGE_TAG -t $IMAGE_TAG -t $LATEST_IMAGE_TAG .
+    - docker push $LATEST_IMAGE_TAG
+    - docker push $IMAGE_TAG
+
+include:
+  - template: Dependency-Scanning.gitlab-ci.yml
+  - template: Container-Scanning.gitlab-ci.yml  
+  - template: SAST.gitlab-ci.yml
+  - template: DAST.gitlab-ci.yml
+
+variables:
+  DS_DISABLE_DIND: "true"
+  SAST_DISABLE_DIND: "true"
+  DAST_WEBSITE: https://packagestest.gentoo.org
+#  DAST_FULL_SCAN_ENABLED: "true"
diff --git a/.travis.yml b/.travis.yml
index b8d7be3..b3b6d1e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 import:
   - source: .travis/travis.yml
     if: branch = master
-  - source: .travis/travis.docker.yml
-    if: branch = dev/main
\ No newline at end of file
+#  - source: .travis/travis.docker.yml
+#    if: branch = dev/main
diff --git a/Dockerfile b/Dockerfile
index 55e6cb0..13d4f94 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,24 @@
-FROM gentoo/rails:latest
+FROM gentoo/portage:latest as portage
+FROM gentoo/stage3-amd64
+
+# Need a portage tree to build, use last nights.
+COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo
+
+# Sandbox doesn't work well in docker.
+ENV FEATURES="-userpriv -usersandbox -sandbox"
+ENV USE="-bindist"
+
+RUN emerge -C openssh
+RUN emerge --quiet-build \
+    net-libs/nodejs \
+    dev-lang/ruby \
+    dev-vcs/git
+RUN ACCEPT_KEYWORDS="~amd64" emerge --quiet-build sys-apps/yarn
+
+RUN eselect ruby set ruby25
+
+# Bundler is how we install the ruby stuff.
+RUN gem install bundler -v 1.17.3
 
 # Needed for changelogs.
 RUN git clone https://anongit.gentoo.org/git/repo/gentoo.git /mnt/packages-tree/gentoo/
@@ -17,3 +37,4 @@ RUN cp /var/www/packages.gentoo.org/htdocs/config/initializers/kkuleomi_config.r
 # Precompile our assets.
 RUN bundle exec rake webpacker:compile
 CMD ["bundler", "exec", "thin", "start"]
+
diff --git a/Gemfile.ci b/Gemfile.ci
deleted file mode 100644
index eb5bb32..0000000
--- a/Gemfile.ci
+++ /dev/null
@@ -1,51 +0,0 @@
-source 'https://rubygems.org'
-
-# Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
-gem 'rails', '5.2.0'
-# Use mysql as the database for Active Record
-# gem 'mysql2'
-# Use SCSS for stylesheets
-gem 'sass-rails', '~> 5.0'
-# Use Uglifier as compressor for JavaScript assets
-gem 'uglifier', '>= 1.3.0'
-
-# Use jquery as the JavaScript library
-gem 'jquery-rails'
-# Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks
-gem 'turbolinks'
-# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
-gem 'jbuilder', '~> 2.0'
-# bundle exec rake doc:rails generates the API under doc/api.
-gem 'sdoc', '~> 0.4.0', group: :doc
-
-# packages stuff
-gem 'elasticsearch-rails'
-gem 'elasticsearch-persistence'
-
-gem 'nokogiri'
-gem 'parallel'
-gem 'ruby-progressbar'
-gem 'git'
-gem 'thin'
-
-gem 'sinatra', require: false
-gem 'sidekiq', require: false
-
-gem 'rdiscount'
-
-# Sidekiq bug?
-gem 'did_you_mean', require: false # Sidekiq missing dep
-
-# UI
-gem 'octicons_helper'
-
-group :development do
-  # Call 'byebug' anywhere in the code to stop execution and get a debugger console
-  gem 'byebug'
-
-  # Access an IRB console on exception pages or by using <%= console %> in views
-  gem 'web-console', '~> 2.0'
-
-  # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
-  gem 'spring'
-end
diff --git a/README.md b/README.md
index 26ec7f4..c42d319 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@ internally codenamed kkuleomi/꾸러미 which is Korean for package (who would h
 
 ## Build status
 
-* [![Travis-CI Build Status](https://travis-ci.org/gentoo/gpackages.svg?branch=master)](https://travis-ci.org/gentoo/gpackages)
+[![Travis-CI Build Status](https://travis-ci.org/gentoo/gpackages.svg?branch=master)](https://travis-ci.org/gentoo/gpackages)
 
 ## Installation instructions
 
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index cc5b772..ce9dda9 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -8,7 +8,7 @@
   <meta name="description" content="<%= "#{@description} in the " if @description %>Gentoo Packages Database">
   <%= stylesheet_pack_tag 'application' %>
   <%= javascript_pack_tag 'application' %>
-  <link rel="icon" href="https://www.gentoo.org/favicon.ico" type="image/x-icon">
+  <link rel="icon" href="/favicon.ico" type="image/x-icon">
   <% if content_for? :head -%>
     <%= yield :head %>
   <% end -%>
@@ -108,6 +108,7 @@
       <div class="col-12 col-md-3">
         <h3 class="footerhead">Questions or comments?</h3>
         Please feel free to <a href="https://www.gentoo.org/inside-gentoo/contact/">contact us</a>.
+        <p class="mt-2"><%= `git describe --tags` %></p>
       </div>
     </div>
     <div class="row">
diff --git a/app/webpack/src/javascript/packages/show.js b/app/webpack/src/javascript/packages/show.js
index c964fdb..d3c63a4 100644
--- a/app/webpack/src/javascript/packages/show.js
+++ b/app/webpack/src/javascript/packages/show.js
@@ -2,7 +2,7 @@ $(function() {
   var atom = $('#package-title').data('atom');
 
   $.ajax({
-    url: '/packages/' + atom + '/changelog'
+    url: '/packages/' + atom + '/changelog.html'
   }).done(function(data) {
     $('#changelog-container').html(data);
     $(document).trigger('kkuleomi:ajax');
diff --git a/bin/first-run b/bin/first-run
index f850cc8..5fbe671 100755
--- a/bin/first-run
+++ b/bin/first-run
@@ -11,10 +11,5 @@ bundle exec rake webpacker:compile RAILS_ENV=production
 bundle exec rake kkuleomi:index:init RAILS_ENV=${1:-development}
 ./bin/update-all.sh ${1:-development}
 
-if [[ "${1:-development}" == "production" ]]
-then
-  crontab -l | { cat; echo "*/10 * * * * /var/www/packages.gentoo.org/htdocs/bin/update-all.sh ${1:-development}"; } | crontab -
-fi
-
 # Finally start the http server when the index is initialized
 bundle exec thin start -p 5000
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index c662d13..73a3538 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,11 +17,9 @@ version: '3.2'
 #
 services:
   http-serving:
-    # Build from Dockerfile in .
-    image: gentoo/gpackages:latest
-    build: .
+    image: ${GPACKAGES_IMAGE:-gentoo/packages:latest}
     ports:
-      - 5000
+      - 127.0.0.1:5000:5000
     volumes:
       - type: volume
         source: portage
@@ -38,14 +36,16 @@ services:
       - RAILS_ENV=production
       - NODE_ENV=production
       - MEMCACHE_URL="memcache:11211"
-      - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50
+      - SECRET_KEY_BASE= ${SECRET_KEY_BASE:-6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50}
+    restart: always
+    labels:
+      com.centurylinklabs.watchtower.enable: "true"
     depends_on:
       - redis
       - elasticsearch
     command: bash -c "cp config/initializers/kkuleomi_config.rb.dist config/initializers/kkuleomi_config.rb && /var/www/packages.gentoo.org/htdocs/bin/first-run production"
   sidekiq:
-    image: gentoo/gpackages:latest
-    build: .
+    image: ${GPACKAGES_IMAGE:-gentoo/packages:latest}
     volumes:
       - type: volume
         source: portage
@@ -60,7 +60,10 @@ services:
       - REDIS_URL=redis://redis:6379
       - MEMCACHE_URL="memcache:11211"
       - ELASTICSEARCH_URL=elasticsearch:9200
-      - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50
+      - SECRET_KEY_BASE= ${SECRET_KEY_BASE:-6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50}
+    restart: always
+    labels:
+      com.centurylinklabs.watchtower.enable: "true"
     depends_on:
       - redis
       - elasticsearch
@@ -68,21 +71,35 @@ services:
       bash -c " cp config/initializers/kkuleomi_config.rb.dist config/initializers/kkuleomi_config.rb && bundler install && yarn install --check-files
       && bundle exec sidekiq -c 5"
   memcache:
-    image: memcached:latest
-    ports:
-      - 11211
+    image: memcached:${MEMCACHED_VERSION:-1.5}
+    restart: always
   elasticsearch:
     # TODO(antarus): We should build a docker image for this based on gentoo.
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.3.1
-    # Run in single-node config.
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-7.3.1}
+    volumes:
+      - ${ELASTICSEARCH_DATA_PATH:-/var/lib/elasticsearch/data/}:/usr/share/elasticsearch/data/
     environment:
-      - discovery.type=single-node
-    ports:
-      - 9200
+      - node.name=packages
+      - cluster.initial_master_nodes=packages
+      - cluster.routing.allocation.node_initial_primaries_recoveries=8
+      - cluster.routing.allocation.node_concurrent_outgoing_recoveries=8
+      - cluster.routing.allocation.node_concurrent_incoming_recoveries=8
+      - indices.recovery.max_bytes_per_sec=200mb
+      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    restart: always
   redis:
-    image: redis:4.0.6
-    ports:
-      - 6379
+    image: redis:${REDIS_VERSION:-4.0.6}
+    restart: always
+  watchtower:
+    image: containrrr/watchtower:0.3.10
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /root/.docker/config.json:/config.json
+    command: --label-enable
 
 volumes:
   portage:
diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb
index 9640a87..ff8cb40 100644
--- a/lib/kkuleomi/store.rb
+++ b/lib/kkuleomi/store.rb
@@ -13,7 +13,7 @@ module Kkuleomi::Store
 
 		# In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own.
 		repositories.each { |repository|
-						repository.instance.create_index!(force: true, settings: settings)
+						repository.instance.create_index!(force: _force, settings: settings)
 		}
   end
 end
diff --git a/lib/tasks/kkuleomi.rake b/lib/tasks/kkuleomi.rake
index 9362b7a..45d9283 100644
--- a/lib/tasks/kkuleomi.rake
+++ b/lib/tasks/kkuleomi.rake
@@ -2,6 +2,10 @@ namespace :kkuleomi do
   namespace :index do
     desc '(Re-)Initializes the ElasticSearch index'
     task init: :environment do
+      Kkuleomi::Store.create_index false
+    end
+
+    task recreate: :environment do
       Kkuleomi::Store.create_index true
     end
   end