aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMax Magorsch <max@magorsch.de>2019-09-05 15:51:34 +0200
committerMax Magorsch <max@magorsch.de>2019-09-05 15:51:34 +0200
commitc62bd4d5644a77d1224f188d092f39e0438062ec (patch)
tree8dc0d218c0dae45add04b1324111318048fa1105
parentAdded docker-compose.override.yml for development purposes (diff)
downloadpackages-5-c62bd4d5644a77d1224f188d092f39e0438062ec.tar.gz
packages-5-c62bd4d5644a77d1224f188d092f39e0438062ec.tar.bz2
packages-5-c62bd4d5644a77d1224f188d092f39e0438062ec.zip
Migrate to ES 7.3 and the repository pattern
Elasticsearch-persistence is used as the persistence layer for Ruby domain objects in Elasticsearch in this application. So far, the ActiveRecord pattern has been used here. However, this pattern has been deprecated as of version 6 of the gem and was removed in version 7. That's why the application has been migrated to use the repository pattern instead. For further information, please see: https://www.elastic.co/blog/activerecord-to-repository-changing- persistence-patterns-with-the-elasticsearch-rails-gem Note: The old Elasticsearch index won't be compatible with this version anymore. That's why a fresh index should be populated. Signed-off-by: Max Magorsch <max@magorsch.de>
-rw-r--r--Gemfile4
-rw-r--r--Gemfile.lock42
-rw-r--r--app/controllers/arches_controller.rb12
-rw-r--r--app/controllers/categories_controller.rb10
-rw-r--r--app/controllers/concerns/package_update_feeds.rb8
-rw-r--r--app/controllers/packages_controller.rb14
-rw-r--r--app/controllers/useflags_controller.rb12
-rw-r--r--app/helpers/application_helper.rb3
-rw-r--r--app/jobs/category_update_job.rb4
-rw-r--r--app/jobs/package_removal_job.rb6
-rw-r--r--app/jobs/package_update_job.rb2
-rw-r--r--app/jobs/record_change_job.rb2
-rw-r--r--app/jobs/useflags_update_job.rb16
-rw-r--r--app/models/category.rb41
-rw-r--r--app/models/change.rb53
-rw-r--r--app/models/package.rb80
-rw-r--r--app/models/useflag.rb125
-rw-r--r--app/models/version.rb67
-rw-r--r--app/repositories/base_repository.rb88
-rw-r--r--app/repositories/category_repository.rb21
-rw-r--r--app/repositories/change_repository.rb23
-rw-r--r--app/repositories/elasticsearch_client.rb13
-rw-r--r--app/repositories/package_repository.rb188
-rw-r--r--app/repositories/useflag_repository.rb99
-rw-r--r--app/repositories/version_repository.rb56
-rw-r--r--app/views/arches/keyworded.html.erb2
-rw-r--r--app/views/arches/stable.html.erb2
-rw-r--r--app/views/feeds/changes.atom.builder2
-rw-r--r--app/views/index/_package.html.erb2
-rw-r--r--app/views/index/index.html.erb4
-rw-r--r--app/views/packages/_metadata.html.erb2
-rw-r--r--app/views/packages/_package_header.html.erb2
-rw-r--r--app/views/packages/added.html.erb2
-rw-r--r--app/views/packages/keyworded.html.erb2
-rw-r--r--app/views/packages/search.html.erb6
-rw-r--r--app/views/packages/stable.html.erb2
-rw-r--r--app/views/packages/updated.html.erb2
-rw-r--r--app/views/useflags/_useflag_result_row.html.erb6
-rw-r--r--config/initializers/elasticsearch.rb7
-rw-r--r--lib/kkuleomi/store.rb27
-rw-r--r--lib/kkuleomi/store/model.rb78
-rw-r--r--lib/kkuleomi/store/models/package_import.rb22
-rw-r--r--lib/kkuleomi/store/models/package_search.rb161
-rw-r--r--lib/kkuleomi/store/models/version_import.rb4
44 files changed, 819 insertions, 505 deletions
diff --git a/Gemfile b/Gemfile
index cdcddd2..fd7147d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -23,8 +23,8 @@ gem 'jbuilder', '~> 2.0'
gem 'sdoc', '~> 1.0', group: :doc
# packages stuff
-gem 'elasticsearch-rails', '~> 5.0'
-gem 'elasticsearch-persistence', '~> 5.0'
+gem 'elasticsearch-rails', '~> 7.0.0'
+gem 'elasticsearch-persistence', '~> 7.0.0'
gem 'nokogiri'
gem 'thin'
diff --git a/Gemfile.lock b/Gemfile.lock
index 78774c0..14ab1e1 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -44,42 +44,32 @@ GEM
tzinfo (~> 1.1)
arel (9.0.0)
ast (2.4.0)
- axiom-types (0.1.1)
- descendants_tracker (~> 0.0.4)
- ice_nine (~> 0.11.0)
- thread_safe (~> 0.3, >= 0.3.1)
bindex (0.8.1)
builder (3.2.3)
byebug (11.0.1)
- coercible (1.0.0)
- descendants_tracker (~> 0.0.1)
concurrent-ruby (1.1.5)
connection_pool (2.2.2)
crass (1.0.4)
daemons (1.3.1)
- descendants_tracker (0.0.4)
- thread_safe (~> 0.3, >= 0.3.1)
- elasticsearch (5.0.5)
- elasticsearch-api (= 5.0.5)
- elasticsearch-transport (= 5.0.5)
- elasticsearch-api (5.0.5)
+ elasticsearch (7.3.0)
+ elasticsearch-api (= 7.3.0)
+ elasticsearch-transport (= 7.3.0)
+ elasticsearch-api (7.3.0)
multi_json
- elasticsearch-model (5.1.0)
+ elasticsearch-model (7.0.0)
activesupport (> 3)
- elasticsearch (~> 5)
+ elasticsearch (> 1)
hashie
- elasticsearch-persistence (5.1.0)
+ elasticsearch-persistence (7.0.0)
activemodel (> 4)
activesupport (> 4)
- elasticsearch (~> 5)
- elasticsearch-model (~> 5)
+ elasticsearch (~> 7)
+ elasticsearch-model (= 7.0.0)
hashie
- virtus
- elasticsearch-rails (5.1.0)
- elasticsearch-transport (5.0.5)
+ elasticsearch-rails (7.0.0)
+ elasticsearch-transport (7.3.0)
faraday
multi_json
- equalizer (0.0.11)
erubi (1.8.0)
eventmachine (1.2.7)
execjs (2.7.0)
@@ -91,7 +81,6 @@ GEM
hashie (3.6.0)
i18n (1.6.0)
concurrent-ruby (~> 1.0)
- ice_nine (0.11.2)
jaro_winkler (1.5.3)
jbuilder (2.9.1)
activesupport (>= 4.2.0)
@@ -222,11 +211,6 @@ GEM
uglifier (4.1.20)
execjs (>= 0.3.0, < 3)
unicode-display_width (1.6.0)
- virtus (1.0.5)
- axiom-types (~> 0.1)
- coercible (~> 1.0)
- descendants_tracker (~> 0.0, >= 0.0.3)
- equalizer (~> 0.0, >= 0.0.9)
web-console (3.7.0)
actionview (>= 5.0)
activemodel (>= 5.0)
@@ -241,8 +225,8 @@ PLATFORMS
DEPENDENCIES
byebug
- elasticsearch-persistence (~> 5.0)
- elasticsearch-rails (~> 5.0)
+ elasticsearch-persistence (~> 7.0.0)
+ elasticsearch-rails (~> 7.0.0)
jbuilder (~> 2.0)
jquery-rails (~> 4.3.5)
listen
diff --git a/app/controllers/arches_controller.rb b/app/controllers/arches_controller.rb
index cbbcb65..c72e378 100644
--- a/app/controllers/arches_controller.rb
+++ b/app/controllers/arches_controller.rb
@@ -43,9 +43,9 @@ class ArchesController < ApplicationController
def keyworded_packages(arch)
Rails.cache.fetch("keyworded_packages/#{arch}", expires_in: 10.minutes) do
- Change.filter_all({ change_type: 'keyword', arches: arch },
- size: 50,
- sort: { created_at: { order: 'desc' } }).map do |change|
+ ChangeRepository.filter_all({ change_type: 'keyword', arches: arch },
+ size: 50,
+ sort: { created_at: { order: 'desc' } }).map do |change|
change.to_os(:change_type, :package, :category, :version, :arches, :created_at)
end
end
@@ -53,9 +53,9 @@ class ArchesController < ApplicationController
def stabled_packages(arch)
Rails.cache.fetch("stabled_packages/#{arch}", expires_in: 10.minutes) do
- Change.filter_all({ change_type: 'stable', arches: arch },
- size: 50,
- sort: { created_at: { order: 'desc' } }).map do |change|
+ ChangeRepository.filter_all({ change_type: 'stable', arches: arch },
+ size: 50,
+ sort: { created_at: { order: 'desc' } }).map do |change|
change.to_os(:change_type, :package, :category, :version, :arches, :created_at)
end
end
diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb
index 33817aa..a9c9b06 100644
--- a/app/controllers/categories_controller.rb
+++ b/app/controllers/categories_controller.rb
@@ -3,15 +3,15 @@ class CategoriesController < ApplicationController
before_action :set_nav
def index
- @categories = Category.all_sorted_by(:name, :asc)
+ @categories = CategoryRepository.all_sorted_by(:id, :asc)
end
def show
@packages = Rails.cache.fetch("category/#{@category.name}/packages",
expires_in: 10.minutes) do
- Package.find_all_by(:category,
- @category.name,
- sort: { name_sort: { order: 'asc' } }).map do |pkg|
+ PackageRepository.find_all_by(:category,
+ @category.name,
+ sort: { name_sort: { order: 'asc' } }).map do |pkg|
pkg.to_os(:name, :atom, :description)
end
end
@@ -24,7 +24,7 @@ class CategoriesController < ApplicationController
private
def set_category
- @category = Category.find_by(:name, params[:id])
+ @category = CategoryRepository.find_by(:name, params[:id])
fail ActionController::RoutingError, 'No such category' unless @category
@title = @category.name
diff --git a/app/controllers/concerns/package_update_feeds.rb b/app/controllers/concerns/package_update_feeds.rb
index 2d20672..28a951b 100644
--- a/app/controllers/concerns/package_update_feeds.rb
+++ b/app/controllers/concerns/package_update_feeds.rb
@@ -3,7 +3,7 @@ module PackageUpdateFeeds
def new_packages
Rails.cache.fetch('new_packages', expires_in: 10.minutes) do
- Change.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
+ ChangeRepository.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
change.to_os(:change_type, :package, :category, :created_at)
end
end
@@ -11,7 +11,7 @@ module PackageUpdateFeeds
def version_bumps
Rails.cache.fetch('version_bumps', expires_in: 10.minutes) do
- Change.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
+ ChangeRepository.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
change.to_os(:change_type, :package, :category, :version, :created_at)
end
end
@@ -19,7 +19,7 @@ module PackageUpdateFeeds
def keyworded_packages
Rails.cache.fetch('keyworded_packages', expires_in: 10.minutes) do
- Change.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
+ ChangeRepository.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
change.to_os(:change_type, :package, :category, :version, :arches, :created_at)
end
end
@@ -27,7 +27,7 @@ module PackageUpdateFeeds
def stabled_packages
Rails.cache.fetch('stabled_packages', expires_in: 10.minutes) do
- Change.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
+ ChangeRepository.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change|
change.to_os(:change_type, :package, :category, :version, :arches, :created_at)
end
end
diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb
index 64cb289..67cc86f 100644
--- a/app/controllers/packages_controller.rb
+++ b/app/controllers/packages_controller.rb
@@ -8,24 +8,24 @@ class PackagesController < ApplicationController
def search
@offset = params[:o].to_i || 0
- @packages = Package.default_search(params[:q], @offset)
+ @packages = PackageRepository.default_search(params[:q], @offset)
redirect_to package_path(@packages.first).gsub('%2F', '/') if @packages.size == 1
end
def suggest
- @packages = Package.suggest(params[:q])
+ @packages = PackageRepository.suggest(params[:q])
end
def resolve
- @packages = Package.resolve(params[:atom])
+ @packages = PackageRepository.resolve(params[:atom])
end
def show
- @package = Package.find_by(:atom, params[:id])
+ @package = PackageRepository.find_by(:atom, params[:id])
fail ActionController::RoutingError, 'No such package' unless @package
- fresh_when etag: @package.updated_at, last_modified: @package.updated_at, public: true
+ fresh_when etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true
# Enable this in 2024 (when we have full-color emojis on a Linux desktop)
# @title = ' &#x1F4E6; %s' % @package.atom
@@ -34,10 +34,10 @@ class PackagesController < ApplicationController
end
def changelog
- @package = Package.find_by(:atom, params[:id])
+ @package = PackageRepository.find_by(:atom, params[:id])
fail ActionController::RoutingError, 'No such package' unless @package
- if stale?(etag: @package.updated_at, last_modified: @package.updated_at, public: true)
+ if stale?(etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true)
@changelog = Rails.cache.fetch("changelog/#{@package.atom}") do
Portage::Util::History.for(@package.category, @package.name, 5)
end
diff --git a/app/controllers/useflags_controller.rb b/app/controllers/useflags_controller.rb
index 0fa74f4..9802b78 100644
--- a/app/controllers/useflags_controller.rb
+++ b/app/controllers/useflags_controller.rb
@@ -6,18 +6,18 @@ class UseflagsController < ApplicationController
end
def show
- @useflags = Useflag.get_flags(params[:id])
+ @useflags = UseflagRepository.get_flags(params[:id])
if @useflags.empty? || (@useflags[:use_expand].empty? && @useflags[:local].empty? && @useflags[:global].empty?)
fail ActionController::RoutingError, 'No such useflag'
end
- @packages = Package.find_atoms_by_useflag(params[:id])
+ @packages = PackageRepository.find_atoms_by_useflag(params[:id])
@title = '%s – %s' % [params[:id], t(:use_flags)]
unless @useflags[:use_expand].empty?
@useflag = @useflags[:use_expand].first
- @use_expand_flags = Useflag.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix)
+ @use_expand_flags = UseflagRepository.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix)
@use_expand_flag_name = @useflag.use_expand_prefix.upcase
render template: 'useflags/show_use_expand'
@@ -29,16 +29,16 @@ class UseflagsController < ApplicationController
def search
# TODO: Different search?
- @flags = Useflag.suggest(params[:q])
+ @flags = UseflagRepository.suggest(params[:q])
end
def suggest
- @flags = Useflag.suggest(params[:q])
+ @flags = UseflagRepository.suggest(params[:q])
end
def popular
@popular_useflags = Rails.cache.fetch('popular_useflags', expires_in: 24.hours) do
- Version.get_popular_useflags(100)
+ VersionRepository.get_popular_useflags(100)
end
end
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 619582c..8405e59 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -37,6 +37,9 @@ module ApplicationHelper
end
def i18n_date(date, format = '%a, %e %b %Y %H:%M')
+
+ date = Time.parse(date).utc if date.is_a? String
+
content_tag :span,
l(date, format: format),
class: 'kk-i18n-date',
diff --git a/app/jobs/category_update_job.rb b/app/jobs/category_update_job.rb
index 7443099..e764ad8 100644
--- a/app/jobs/category_update_job.rb
+++ b/app/jobs/category_update_job.rb
@@ -5,8 +5,8 @@ class CategoryUpdateJob < ApplicationJob
category_path, options = args
model = Portage::Repository::Category.new(category_path)
- category = Category.find_by(:name, model.name) || Category.new
- idx_packages = Package.find_all_by(:category, model.name) || []
+ category = CategoryRepository.find_by(:name, model.name) || Category.new
+ idx_packages = PackageRepository.find_all_by(:category, model.name) || []
if category.needs_import? model
category.import! model
diff --git a/app/jobs/package_removal_job.rb b/app/jobs/package_removal_job.rb
index 877ed07..e625b96 100644
--- a/app/jobs/package_removal_job.rb
+++ b/app/jobs/package_removal_job.rb
@@ -4,11 +4,11 @@ class PackageRemovalJob < ApplicationJob
def perform(*args)
atom, _options = args
- package_doc = Package.find_by(:atom, atom)
+ package_doc = PackageRepository.find_by(:atom, atom)
return if package_doc.nil?
- package_doc.versions.each(&:delete)
- package_doc.delete
+ package_doc.versions.each { |v| VersionRepository.delete(v) }
+ PackageRepository.delete(package_doc)
Rails.logger.warn { "Package deleted: #{atom}" }
# USE flags are cleaned up by the UseflagsUpdateJob
diff --git a/app/jobs/package_update_job.rb b/app/jobs/package_update_job.rb
index 55e278f..53a352c 100644
--- a/app/jobs/package_update_job.rb
+++ b/app/jobs/package_update_job.rb
@@ -4,7 +4,7 @@ class PackageUpdateJob < ApplicationJob
def perform(*args)
path, options = args
package_model = Portage::Repository::Package.new(path)
- package_doc = Package.find_by(:atom, package_model.to_cp) || Package.new
+ package_doc = PackageRepository.find_by(:atom, package_model.to_cp) || Package.new
if package_doc.needs_import? package_model
package_doc.import!(package_model, options)
diff --git a/app/jobs/record_change_job.rb b/app/jobs/record_change_job.rb
index 0e6a011..ed5dd5e 100644
--- a/app/jobs/record_change_job.rb
+++ b/app/jobs/record_change_job.rb
@@ -25,6 +25,6 @@ class RecordChangeJob < ApplicationJob
c.change_type = 'removal'
end
- c.save
+ ChangeRepository.save(c)
end
end
diff --git a/app/jobs/useflags_update_job.rb b/app/jobs/useflags_update_job.rb
index 21145c3..5558d47 100644
--- a/app/jobs/useflags_update_job.rb
+++ b/app/jobs/useflags_update_job.rb
@@ -10,7 +10,7 @@ class UseflagsUpdateJob < ApplicationJob
def update_global(repo)
model_flags = repo.global_useflags
- index_flags = Useflag.global
+ index_flags = UseflagRepository.global
new_flags = model_flags.keys - index_flags.keys
del_flags = index_flags.keys - model_flags.keys
@@ -21,24 +21,24 @@ class UseflagsUpdateJob < ApplicationJob
flag_doc.name = flag
flag_doc.description = model_flags[flag]
flag_doc.scope = 'global'
- flag_doc.save
+ UseflagRepository.save(flag_doc)
end
eql_flags.each do |flag|
unless index_flags[flag].description == model_flags[flag]
index_flags[flag].description = model_flags[flag]
- index_flags[flag].save
+ UseflagRepository.save(index_flags[flag])
end
end
del_flags.each do |flag|
- index_flags[flag].delete
+ UseflagRepository.delete(index_flags[flag])
end
end
def update_use_expand(repo)
model_flags = repo.use_expand_flags
- index_flags = Useflag.use_expand
+ index_flags = UseflagRepository.use_expand
# Calculate keys only once
index_flag_keys = index_flags.keys
@@ -55,7 +55,7 @@ class UseflagsUpdateJob < ApplicationJob
if index_flag_keys.include? _flag
unless index_flags[_flag].description == desc
index_flags[_flag].description = desc
- index_flags[_flag].save
+ UseflagRepository.save(index_flags[_flag])
end
else
# New flag
@@ -64,14 +64,14 @@ class UseflagsUpdateJob < ApplicationJob
flag_doc.description = desc
flag_doc.scope = 'use_expand'
flag_doc.use_expand_prefix = variable
- flag_doc.save
+ UseflagRepository.save(flag_doc)
end
end
end
# Find and process removed flags
flag_status.each_pair do |flag, status|
- index_flags[flag].delete unless status
+ UseflagRepository.delete(index_flags[flag]) unless status
end
end
diff --git a/app/models/category.rb b/app/models/category.rb
index f629bde..4e361c1 100644
--- a/app/models/category.rb
+++ b/app/models/category.rb
@@ -1,12 +1,39 @@
class Category
- include Elasticsearch::Persistence::Model
- include Kkuleomi::Store::Model
+ include ActiveModel::Model
+ include ActiveModel::Validations
- index_name "categories-#{Rails.env}"
+ ATTRIBUTES = [:id,
+ :created_at,
+ :updated_at,
+ :name,
+ :description,
+ :metadata_hash]
+ attr_accessor(*ATTRIBUTES)
+ attr_reader :attributes
+
+ validates :name, presence: true
+
+ def initialize(attr={})
+ attr.each do |k,v|
+ if ATTRIBUTES.include?(k.to_sym)
+ send("#{k}=", v)
+ end
+ end
+ end
+
+ def attributes
+ @id = @name
+ @created_at ||= DateTime.now
+ @updated_at = DateTime.now
+ ATTRIBUTES.inject({}) do |hash, attr|
+ if value = send(attr)
+ hash[attr] = value
+ end
+ hash
+ end
+ end
+ alias :to_hash :attributes
- attribute :name, String, mapping: { type: 'keyword' }
- attribute :description, String, mapping: { type: 'text' }
- attribute :metadata_hash, String, mapping: { type: 'text' }
# Determines if the document model needs an update from the repository model
#
@@ -29,7 +56,7 @@ class Category
# @param [Portage::Repository::Category] category_model Input category model
def import!(category_model)
import(category_model)
- save
+ CategoryRepository.save(self)
end
# Returns the URL parameter for referencing this package (Rails internal stuff)
diff --git a/app/models/change.rb b/app/models/change.rb
index 6eaf00c..1793da4 100644
--- a/app/models/change.rb
+++ b/app/models/change.rb
@@ -1,13 +1,48 @@
class Change
- include Elasticsearch::Persistence::Model
- include Kkuleomi::Store::Model
+ include ActiveModel::Model
+ include ActiveModel::Validations
- index_name "change-#{Rails.env}"
+ ATTRIBUTES = [:_id,
+ :created_at,
+ :updated_at,
+ :package,
+ :category,
+ :change_type,
+ :version,
+ :arches,
+ :commit]
+ attr_accessor(*ATTRIBUTES)
+ attr_reader :attributes
+
+ validates :package, presence: true
+
+ def initialize(attr={})
+ attr.each do |k,v|
+ if ATTRIBUTES.include?(k.to_sym)
+ send("#{k}=", v)
+ end
+ end
+ end
+
+ def attributes
+ @created_at ||= DateTime.now
+ @updated_at = DateTime.now
+ ATTRIBUTES.inject({}) do |hash, attr|
+ if value = send(attr)
+ hash[attr] = value
+ end
+ hash
+ end
+ end
+ alias :to_hash :attributes
+
+ # Converts the model to an OpenStruct instance
+ #
+ # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil
+ # @return [OpenStruct] OpenStruct containing the selected fields
+ def to_os(*fields)
+ fields = all_fields if fields.empty?
+ OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }])
+ end
- attribute :package, String, mapping: { type: 'keyword' }
- attribute :category, String, mapping: { type: 'keyword' }
- attribute :change_type, String, mapping: { type: 'keyword' }
- attribute :version, String, mapping: { type: 'keyword' }
- attribute :arches, String, mapping: { type: 'keyword' }
- attribute :commit, Hash, default: {}, mapping: { type: 'object' }
end
diff --git a/app/models/package.rb b/app/models/package.rb
index 7ad3cbe..11ef135 100644
--- a/app/models/package.rb
+++ b/app/models/package.rb
@@ -1,31 +1,52 @@
class Package
- include Elasticsearch::Persistence::Model
- include Kkuleomi::Store::Model
+ include ActiveModel::Model
+ include ActiveModel::Validations
include Kkuleomi::Store::Models::PackageImport
- include Kkuleomi::Store::Models::PackageSearch
-
- index_name "packages-#{Rails.env}"
-
- raw_fields = {
- type: 'keyword'
- }
-
- attribute :category, String, mapping: raw_fields
- attribute :name, String, mapping: raw_fields
- attribute :name_sort, String, mapping: raw_fields
- attribute :atom, String, mapping: raw_fields
- attribute :description, String, mapping: { type: 'text' }
- attribute :longdescription, String, mapping: { type: 'text' }
- attribute :homepage, String, default: [], mapping: raw_fields
- attribute :license, String, mapping: raw_fields
- attribute :licenses, String, default: [], mapping: raw_fields
- attribute :herds, String, default: [], mapping: raw_fields
- attribute :maintainers, Array, default: [], mapping: { type: 'object' }
- attribute :useflags, Hash, default: {}, mapping: { type: 'object' }
- attribute :metadata_hash, String, mapping: raw_fields
+
+ ATTRIBUTES = [:id,
+ :created_at,
+ :updated_at,
+ :category,
+ :name,
+ :name_sort,
+ :atom,
+ :description,
+ :longdescription,
+ :homepage,
+ :license,
+ :licenses,
+ :herds,
+ :maintainers,
+ :useflags,
+ :metadata_hash]
+ attr_accessor(*ATTRIBUTES)
+ attr_reader :attributes
+
+ validates :name, presence: true
+
+ def initialize(attr={})
+ attr.each do |k,v|
+ if ATTRIBUTES.include?(k.to_sym)
+ send("#{k}=", v)
+ end
+ end
+ end
+
+ def attributes
+ @id = @atom
+ @created_at ||= DateTime.now
+ @updated_at = DateTime.now
+ ATTRIBUTES.inject({}) do |hash, attr|
+ if value = send(attr)
+ hash[attr] = value
+ end
+ hash
+ end
+ end
+ alias :to_hash :attributes
def category_model
- @category_model ||= Category.find_by(:name, category)
+ @category_model ||= CategoryRepository.find_by(:name, category)
end
def to_param
@@ -44,7 +65,7 @@ class Package
end
def versions
- @versions ||= Version.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } })
+ @versions ||= VersionRepository.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } })
end
def latest_version
@@ -65,6 +86,15 @@ class Package
maintainers.empty? && herds.empty?
end
+ # Converts the model to an OpenStruct instance
+ #
+ # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil
+ # @return [OpenStruct] OpenStruct containing the selected fields
+ def to_os(*fields)
+ fields = all_fields if fields.empty?
+ OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }])
+ end
+
private
# Splits a license string into single licenses, stripping the permitted logic constructs
diff --git a/app/models/useflag.rb b/app/models/useflag.rb
index 131a89c..12758cb 100644
--- a/app/models/useflag.rb
+++ b/app/models/useflag.rb
@@ -1,14 +1,41 @@
class Useflag
- include Elasticsearch::Persistence::Model
- include Kkuleomi::Store::Model
-
- index_name "useflags-#{Rails.env}"
+ include ActiveModel::Model
+ include ActiveModel::Validations
+
+ ATTRIBUTES = [:id,
+ :created_at,
+ :updated_at,
+ :name,
+ :description,
+ :atom,
+ :scope,
+ :use_expand_prefix]
+ attr_accessor(*ATTRIBUTES)
+ attr_reader :attributes
+
+ validates :name, presence: true
+
+
+ def initialize(attr={})
+ attr.each do |k,v|
+ if ATTRIBUTES.include?(k.to_sym)
+ send("#{k}=", v)
+ end
+ end
+ end
- attribute :name, String, mapping: { type: 'keyword' }
- attribute :description, String, mapping: { type: 'text' }
- attribute :atom, String, mapping: { type: 'keyword' }
- attribute :scope, String, mapping: { type: 'keyword' }
- attribute :use_expand_prefix, String, mapping: { type: 'keyword' }
+ def attributes
+ @id = @name + '-' + (@atom || 'global' ) + '-' + @scope
+ @created_at ||= DateTime.now
+ @updated_at = DateTime.now
+ ATTRIBUTES.inject({}) do |hash, attr|
+ if value = send(attr)
+ hash[attr] = value
+ end
+ hash
+ end
+ end
+ alias :to_hash :attributes
def all_fields
[:name, :description, :atom, :scope, :use_expand_prefix]
@@ -22,78 +49,14 @@ class Useflag
name.gsub(use_expand_prefix + '_', '')
end
- class << self
- # Retrieves all flags sorted by their state
- def get_flags(name)
- result = { local: {}, global: [], use_expand: [] }
-
- find_all_by(:name, name).each do |flag|
- case flag.scope
- when 'local'
- result[:local][flag.atom] = flag
- when 'global'
- result[:global] << flag
- when 'use_expand'
- result[:use_expand] << flag
- end
- end
-
- result
- end
-
- def suggest(q)
- results = Useflag.search(
- size: 20,
- query: { match_phrase_prefix: { name: q } }
- )
-
- processed_results = {}
- results.each do |result|
- if processed_results.key? result.name
- processed_results[result.name] = {
- name: result.name,
- description: '(multiple definitions)',
- scope: 'multi'
- }
- else
- processed_results[result.name] = result
- end
- end
-
- processed_results.values.sort { |a, b| a[:name].length <=> b[:name].length }
- end
-
- # Loads the local USE flags for a given package in a name -> model hash
- #
- # @param [String] atom Package to find flags for
- # @return [Hash]
- def local_for(atom)
- map_by_name find_all_by(:atom, atom)
- end
-
- # Maps the global USE flags in the index by their name
- # This is expensive!
- #
- def global
- map_by_name find_all_by(:scope, 'global')
- end
-
- # Maps the USE_EXPAND variables in the index by their name
- #
- def use_expand
- map_by_name find_all_by(:scope, 'use_expand')
- end
-
- private
+ # Converts the model to a Hash
+ #
+ # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil
+ # @return [Hash] Hash containing the selected fields
+ def to_hsh(*fields)
+ fields = all_fields if fields.empty?
+ Hash[fields.map { |field| [field, send(field)] }]
+ end
- def map_by_name(collection)
- map = {}
- collection.each do |item|
- map[item.name] = item
- end
-
- map
- end
- end
end
diff --git a/app/models/version.rb b/app/models/version.rb
index 62c72f8..3629b98 100644
--- a/app/models/version.rb
+++ b/app/models/version.rb
@@ -1,23 +1,52 @@
+require 'date'
+
class Version
- include Elasticsearch::Persistence::Model
- include Kkuleomi::Store::Model
+ include ActiveModel::Model
+ include ActiveModel::Validations
include Kkuleomi::Store::Models::VersionImport
- index_name "versions-#{Rails.env}"
-
- attribute :version, String, mapping: { type: 'keyword' }
- attribute :package, String, mapping: { type: 'keyword' }
- attribute :atom, String, mapping: { type: 'keyword' }
- attribute :sort_key, Integer, mapping: { type: 'integer' }
- attribute :slot, String, mapping: { type: 'keyword' }
- attribute :subslot, String, mapping: { type: 'keyword' }
- attribute :eapi, String, mapping: { type: 'keyword' }
- attribute :keywords, String, mapping: { type: 'keyword' }
- attribute :masks, Array, default: [], mapping: { type: 'object' }
- attribute :use, String, default: [], mapping: { type: 'keyword' }
- attribute :restrict, String, default: [], mapping: { type: 'keyword' }
- attribute :properties, String, default: [], mapping: { type: 'keyword' }
- attribute :metadata_hash, String, mapping: { type: 'keyword' }
+ ATTRIBUTES = [:id,
+ :created_at,
+ :updated_at,
+ :version,
+ :package,
+ :atom,
+ :sort_key,
+ :slot,
+ :subslot,
+ :eapi,
+ :keywords,
+ :masks,
+ :use,
+ :restrict,
+ :properties,
+ :metadata_hash]
+ attr_accessor(*ATTRIBUTES)
+ attr_reader :attributes
+
+ validates :version, presence: true
+
+ def initialize(attr={})
+ attr.each do |k,v|
+ if ATTRIBUTES.include?(k.to_sym)
+ send("#{k}=", v)
+ end
+ end
+ end
+
+ def attributes
+ @id = @atom
+ @created_at ||= DateTime.now
+ @updated_at = DateTime.now
+
+ ATTRIBUTES.inject({}) do |hash, attr|
+ if value = send(attr)
+ hash[attr] = value
+ end
+ hash
+ end
+ end
+ alias :to_hash :attributes
# Returns the keywording state on a given architecture
#
@@ -138,14 +167,14 @@ class Version
def calc_useflags
result = { local: {}, global: {}, use_expand: {} }
- local_flag_map = Useflag.local_for(atom.gsub("-#{version}", ''))
+ local_flag_map = UseflagRepository.local_for(atom.gsub("-#{version}", ''))
local_flags = local_flag_map.keys
use.sort.each do |flag|
if local_flags.include? flag
result[:local][flag] = local_flag_map[flag].to_hsh
else
- useflag = Useflag.find_by(:name, flag)
+ useflag = UseflagRepository.find_by(:name, flag)
# This should not happen, but let's be sure
next unless useflag
diff --git a/app/repositories/base_repository.rb b/app/repositories/base_repository.rb
new file mode 100644
index 0000000..7154691
--- /dev/null
+++ b/app/repositories/base_repository.rb
@@ -0,0 +1,88 @@
+require 'forwardable'
+require 'singleton'
+
+class BaseRepository
+ include Elasticsearch::Persistence::Repository
+ include Elasticsearch::Persistence::Repository::DSL
+ include Singleton
+
+ client ElasticsearchClient.default
+
+ class << self
+ extend Forwardable
+ def_delegators :instance, :find_all_by, :filter_all, :find_by, :find_all_by_parent, :all_sorted_by
+ def_delegators :instance, :count, :search, :delete, :save, :refresh_index!, :create_index
+ end
+
+ # Finds instances by exact IDs using the 'term' filter
+ def find_all_by(field, value, opts = {})
+ search({
+ size: 10_000,
+ query: { match: { field => value } }
+ }.merge(opts))
+ end
+
+ # Filter all instances by the given parameters
+ def filter_all(filters, opts = {})
+ filter_args = []
+ filters.each_pair { |field, value| filter_args << { term: { field => value } } }
+
+ search({
+ query: {
+ bool: { filter: { bool: { must: filter_args } } }
+ },
+ size: 10_000
+ }.merge(opts))
+ end
+
+ def find_by(field, value, opts = {})
+ find_all_by(field, value, opts).first
+ end
+
+ def find_all_by_parent(parent, opts = {})
+ search(opts.merge(
+ size: 10_000,
+ query: {
+ bool: {
+ filter: {
+ has_parent: {
+ parent_type: parent.class.document_type,
+ query: { term: { _id: parent.id } }
+ }
+ },
+ must: {
+ match_all: {}
+ }
+ }
+ })
+ )
+ end
+
+ # Returns all (by default 10k) records of this class sorted by a field.
+ def all_sorted_by(field, order, options = {})
+ search({
+ size: 10_000,
+ query: { match_all: {} },
+ sort: { field => { order: order } }
+ }.merge(options))
+ end
+
+ # Converts the model to an OpenStruct instance
+ #
+ # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil
+ # @return [OpenStruct] OpenStruct containing the selected fields
+ def to_os(*fields)
+ fields = all_fields if fields.empty?
+ OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }])
+ end
+
+ # Converts the model to a Hash
+ #
+ # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil
+ # @return [Hash] Hash containing the selected fields
+ def to_hsh(*fields)
+ fields = all_fields if fields.empty?
+ Hash[fields.map { |field| [field, send(field)] }]
+ end
+
+end \ No newline at end of file
diff --git a/app/repositories/category_repository.rb b/app/repositories/category_repository.rb
new file mode 100644
index 0000000..e9cf033
--- /dev/null
+++ b/app/repositories/category_repository.rb
@@ -0,0 +1,21 @@
+require 'singleton'
+
+class CategoryRepository < BaseRepository
+ include Singleton
+
+ client ElasticsearchClient.default
+
+ index_name "categories-#{Rails.env}"
+
+ klass Category
+
+ mapping do
+ indexes :id, type: 'keyword'
+ indexes :name, type: 'text'
+ indexes :description, type: 'text'
+ indexes :metadata_hash, type: 'keyword'
+ indexes :created_at, type: 'date'
+ indexes :updated_at, type: 'date'
+ end
+
+end
diff --git a/app/repositories/change_repository.rb b/app/repositories/change_repository.rb
new file mode 100644
index 0000000..e5cc2f2
--- /dev/null
+++ b/app/repositories/change_repository.rb
@@ -0,0 +1,23 @@
+require 'singleton'
+
+class ChangeRepository < BaseRepository
+ include Singleton
+
+ client ElasticsearchClient.default
+
+ index_name "change-#{Rails.env}"
+
+ klass Change
+
+ mapping do
+ indexes :package, type: 'keyword'
+ indexes :category, type: 'keyword'
+ indexes :change_type, type: 'keyword'
+ indexes :version, type: 'keyword'
+ indexes :arches, type: 'keyword'
+ indexes :commit, type: 'object'
+ indexes :created_at, type: 'date'
+ indexes :updated_at, type: 'date'
+ end
+
+end
diff --git a/app/repositories/elasticsearch_client.rb b/app/repositories/elasticsearch_client.rb
new file mode 100644
index 0000000..88de0c8
--- /dev/null
+++ b/app/repositories/elasticsearch_client.rb
@@ -0,0 +1,13 @@
+class ElasticsearchClient
+
+ def self.default
+ @default ||= Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200'
+ end
+
+ private
+
+ def initialize(*)
+ raise "Should not be initialiazed"
+ end
+
+end \ No newline at end of file
diff --git a/app/repositories/package_repository.rb b/app/repositories/package_repository.rb
new file mode 100644
index 0000000..ed77afb
--- /dev/null
+++ b/app/repositories/package_repository.rb
@@ -0,0 +1,188 @@
+require 'forwardable'
+require 'singleton'
+
+class PackageRepository < BaseRepository
+ include Singleton
+
+ class << self
+ extend Forwardable
+ def_delegators :instance, :suggest, :resolve, :find_atoms_by_useflag, :default_search_size, :default_search,
+ :build_query, :match_wildcard, :match_phrase, :match_description, :match_category, :scoring_functions
+ end
+
+ index_name "packages-#{Rails.env}"
+
+ klass Package
+
+ mapping do
+ indexes :category, type: 'keyword'
+ indexes :name, type: 'keyword'
+ indexes :name_sort, type: 'keyword'
+ indexes :atom, type: 'keyword'
+ indexes :description, type: 'text'
+ indexes :longdescription, type: 'text'
+ indexes :homepage, type: 'keyword'
+ indexes :license, type: 'keyword'
+ indexes :licenses, type: 'keyword'
+ indexes :herds, type: 'keyword'
+ indexes :maintainers, type: 'object'
+ indexes :useflags, type: 'object'
+ indexes :metadata_hash, type: 'keyword'
+ indexes :created_at, type: 'date'
+ indexes :updated_at, type: 'date'
+ end
+
+ def suggest(q)
+ PackageRepository.search(
+ size: 20,
+ query: {
+ wildcard: {
+ name_sort: {
+ wildcard: q.downcase + '*'
+ }
+ }
+ }
+ )
+ end
+
+ # Tries to resolve a query atom to one or more packages
+ def resolve(atom)
+ [] if atom.nil? || atom.empty?
+
+ PackageRepository.find_all_by(:atom, atom) + PackageRepository.find_all_by(:name, atom)
+ end
+
+ # Searches the versions index for versions using a certain USE flag.
+ # Results are aggregated by package atoms.
+ def find_atoms_by_useflag(useflag)
+ VersionRepository.search(
+ size: 0, # collect all packages.
+ query: {
+ bool: {
+ must: { match_all: {} },
+ filter: { term: { use: useflag } }
+ }
+ },
+ aggs: {
+ group_by_package: {
+ terms: {
+ field: 'package',
+ order: { '_key' => 'asc' },
+ # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html
+ # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see.
+ # Other limits in this app are also 10k mostly to 'make things fit kinda'.
+ size: 10000,
+ }
+ }
+ },
+ ).response.aggregations['group_by_package'].buckets
+ end
+
+ def default_search_size
+ 25
+ end
+
+ def default_search(q, offset)
+ return [] if q.nil? || q.empty?
+
+ part1, part2 = q.split('/', 2)
+
+ if part2.nil?
+ search(build_query(part1, nil, default_search_size, offset))
+ else
+ search(build_query(part2, part1, default_search_size, offset))
+ end
+ end
+
+ def build_query(q, category, size, offset)
+ {
+ size: size,
+ from: offset,
+ query: {
+ function_score: {
+ query: { bool: bool_query_parts(q, category) },
+ functions: scoring_functions
+ }
+ }
+ }
+ end
+
+ def bool_query_parts(q, category = nil)
+ q_dwncsd = q.downcase
+
+ query = {
+ must: [
+ match_wildcard(q_dwncsd)
+ ],
+ should: [
+ match_phrase(q_dwncsd),
+ match_description(q)
+ ]
+ }
+
+ query[:must] << [match_category(category)] if category
+
+ query
+ end
+
+ def match_wildcard(q)
+ q = ('*' + q + '*') unless q.include? '*'
+ q.tr!(' ', '*')
+
+ {
+ wildcard: {
+ name_sort: {
+ wildcard: q,
+ boost: 4
+ }
+ }
+ }
+ end
+
+ def match_phrase(q)
+ {
+ match_phrase: {
+ name: {
+ query: q,
+ boost: 5
+ }
+ }
+ }
+ end
+
+ def match_description(q)
+ {
+ match: {
+ description: {
+ query: q,
+ boost: 0.1
+ }
+ }
+ }
+ end
+
+ def match_category(cat)
+ {
+ match: {
+ category: {
+ query: cat,
+ boost: 2
+ }
+ }
+ }
+ end
+
+ def scoring_functions
+ [
+ {
+ filter: {
+ term: {
+ category: 'virtual'
+ }
+ },
+ weight: 0.6
+ }
+ ]
+ end
+
+end
diff --git a/app/repositories/useflag_repository.rb b/app/repositories/useflag_repository.rb
new file mode 100644
index 0000000..26328f4
--- /dev/null
+++ b/app/repositories/useflag_repository.rb
@@ -0,0 +1,99 @@
+require 'singleton'
+
+class UseflagRepository < BaseRepository
+ include Singleton
+
+ class << self
+ extend Forwardable
+ def_delegators :instance, :get_flags, :suggest, :local_for, :global, :use_expand
+ end
+
+ index_name "useflags-#{Rails.env}"
+
+ klass Useflag
+
+ mapping do
+ indexes :name, type: 'text'
+ indexes :description, type: 'text'
+ indexes :atom, type: 'keyword'
+ indexes :scope, type: 'keyword'
+ indexes :use_expand_prefix, type: 'keyword'
+ indexes :created_at, type: 'date'
+ indexes :updated_at, type: 'date'
+ end
+
+
+ # Retrieves all flags sorted by their state
+ def get_flags(name)
+ result = { local: {}, global: [], use_expand: [] }
+
+ find_all_by(:name, name).each do |flag|
+ case flag.scope
+ when 'local'
+ result[:local][flag.atom] = flag
+ when 'global'
+ result[:global] << flag
+ when 'use_expand'
+ result[:use_expand] << flag
+ end
+ end
+
+ result
+ end
+
+ def suggest(q)
+ results = search(
+ size: 20,
+ query: { match_phrase_prefix: { name: q } }
+ )
+
+ processed_results = {}
+ results.each do |result|
+ if processed_results.key? result.name
+ processed_results[result.name] = {
+ name: result.name,
+ description: '(multiple definitions)',
+ scope: 'multi'
+ }
+ else
+ processed_results[result.name] = result
+ end
+ end
+
+ processed_results.values.sort { |a, b| a.name.length <=> b.name.length }
+ end
+
+ # Loads the local USE flags for a given package in a name -> model hash
+ #
+ # @param [String] atom Package to find flags for
+ # @return [Hash]
+ def local_for(atom)
+ map_by_name find_all_by(:atom, atom)
+ end
+
+ # Maps the global USE flags in the index by their name
+ # This is expensive!
+ #
+ def global
+ map_by_name find_all_by(:scope, 'global')
+ end
+
+ # Maps the USE_EXPAND variables in the index by their name
+ #
+ def use_expand
+ map_by_name find_all_by(:scope, 'use_expand')
+ end
+
+ private
+
+ def map_by_name(collection)
+ map = {}
+
+ collection.each do |item|
+ map[item.name] = item
+ end
+
+ map
+ end
+
+end
diff --git a/app/repositories/version_repository.rb b/app/repositories/version_repository.rb
new file mode 100644
index 0000000..43168d2
--- /dev/null
+++ b/app/repositories/version_repository.rb
@@ -0,0 +1,56 @@
+require 'singleton'
+
+class VersionRepository < BaseRepository
+ include Singleton
+
+ class << self
+ extend Forwardable
+ def_delegators :instance, :get_popular_useflags
+ end
+
+ index_name "versions-#{Rails.env}"
+
+ klass Version
+
+ mapping do
+ indexes :version, type: 'keyword'
+ indexes :package, type: 'keyword'
+ indexes :atom, type: 'keyword'
+ indexes :sort_key, type: 'integer'
+ indexes :slot, type: 'keyword'
+ indexes :subslot, type: 'keyword'
+ indexes :eapi, type: 'keyword'
+ indexes :keywords, type: 'keyword'
+ indexes :masks do
+ indexes :arches, type: 'keyword'
+ indexes :atoms, type: 'keyword'
+ indexes :author, type: 'keyword'
+ indexes :date, type: 'keyword'
+ indexes :reason, type: 'text'
+ end
+ indexes :use, type: 'keyword'
+ indexes :restrict, type: 'keyword'
+ indexes :properties, type: 'keyword'
+ indexes :metadata_hash, type: 'keyword'
+ indexes :created_at, type: 'date'
+ indexes :updated_at, type: 'date'
+ end
+
+ # Retrieves the most widely used USE flags by all versions
+ # Note that packages with many versions are over-represented
+ def get_popular_useflags(n = 50)
+ search(
+ query: { match_all: {} },
+ aggs: {
+ group_by_flag: {
+ terms: {
+ field: 'use',
+ size: n
+ }
+ }
+ },
+ size: 0
+ ).response.aggregations['group_by_flag'].buckets
+ end
+
+end
diff --git a/app/views/arches/keyworded.html.erb b/app/views/arches/keyworded.html.erb
index b7ae03d..ae1df29 100644
--- a/app/views/arches/keyworded.html.erb
+++ b/app/views/arches/keyworded.html.erb
@@ -12,7 +12,7 @@
<% cache("keyworded-full-#{@arch}-#{@changes.hash}") do %>
<ul class="list-group">
<% @changes.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %>
<% end %>
</ul>
diff --git a/app/views/arches/stable.html.erb b/app/views/arches/stable.html.erb
index b1a4548..eb66245 100644
--- a/app/views/arches/stable.html.erb
+++ b/app/views/arches/stable.html.erb
@@ -12,7 +12,7 @@
<% cache("stable-full-#{@arch}-#{@changes.hash}") do %>
<ul class="list-group">
<% @changes.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %>
<% end %>
</ul>
diff --git a/app/views/feeds/changes.atom.builder b/app/views/feeds/changes.atom.builder
index 5991f45..a8af3df 100644
--- a/app/views/feeds/changes.atom.builder
+++ b/app/views/feeds/changes.atom.builder
@@ -10,7 +10,7 @@ atom_feed(id: atom_id(@feed_type, @feed_id, 'feed')) do |feed|
@changes.each do |change|
atom = cp_to_atom change.category, change.package
- package = Package.find_by :atom, atom
+ package = PackageRepository.find_by :atom, atom
if package.nil?
logger.warn "Package for change (#{change}) nil!"
next
diff --git a/app/views/index/_package.html.erb b/app/views/index/_package.html.erb
index eeb3109..a364209 100644
--- a/app/views/index/_package.html.erb
+++ b/app/views/index/_package.html.erb
@@ -1,4 +1,4 @@
-<%- package = Package.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%>
+<%- package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%>
<tr>
<td>
<a href="<%= slf(package_path(cp_to_atom(change.category, change.package))) %>">
diff --git a/app/views/index/index.html.erb b/app/views/index/index.html.erb
index 890a5f3..af86c9e 100644
--- a/app/views/index/index.html.erb
+++ b/app/views/index/index.html.erb
@@ -1,5 +1,5 @@
<div class="jumbotron">
- <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter Package.count %></span> Gentoo Packages</h2>
+ <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter PackageRepository.count %></span> Gentoo Packages</h2>
<form action="<%= search_packages_path %>" method="get">
<div class="typeahead-container">
@@ -43,7 +43,7 @@
</div>
<ul class="list-group">
<% @version_bumps.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %>
<% end %>
</ul>
diff --git a/app/views/packages/_metadata.html.erb b/app/views/packages/_metadata.html.erb
index 426afd9..5568c08 100644
--- a/app/views/packages/_metadata.html.erb
+++ b/app/views/packages/_metadata.html.erb
@@ -3,7 +3,7 @@
<h3 class="panel-title"><%= t :box_metadata %></h3>
</div>
<ul class="list-group kk-metadata-list">
- <% if package.homepage.size > 1 %>
+ <% if !package.homepage.nil? && package.homepage.size > 1 %>
<li class="kk-metadata-item list-group-item">
<div class="row">
<div class="col-xs-12 col-md-3 kk-metadata-key">
diff --git a/app/views/packages/_package_header.html.erb b/app/views/packages/_package_header.html.erb
index 1b7876b..8c611da 100644
--- a/app/views/packages/_package_header.html.erb
+++ b/app/views/packages/_package_header.html.erb
@@ -25,7 +25,7 @@
<%= package.description %>
</p>
- <% unless package.homepage.empty? || package.homepage.first.nil? || package.homepage.first.empty? %>
+ <% unless package.homepage.nil? || package.homepage.first.nil? || package.homepage.first.empty? %>
<p class="kk-package-homepage">
<%= content_tag :a, package.homepage.first, href: package.homepage.first, rel: 'nofollow' %>
</p>
diff --git a/app/views/packages/added.html.erb b/app/views/packages/added.html.erb
index 97d5cb6..589226a 100644
--- a/app/views/packages/added.html.erb
+++ b/app/views/packages/added.html.erb
@@ -12,7 +12,7 @@
<% cache("added-full-#{@changes.hash}") do %>
<ul class="list-group">
<% @changes.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.latest_version } %>
<% end %>
</ul>
diff --git a/app/views/packages/keyworded.html.erb b/app/views/packages/keyworded.html.erb
index ff5b60c..a83a558 100644
--- a/app/views/packages/keyworded.html.erb
+++ b/app/views/packages/keyworded.html.erb
@@ -12,7 +12,7 @@
<% cache("keyworded-full-#{@changes.hash}") do %>
<ul class="list-group">
<% @changes.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %>
<% end %>
</ul>
diff --git a/app/views/packages/search.html.erb b/app/views/packages/search.html.erb
index fe77dd3..c4b5a75 100644
--- a/app/views/packages/search.html.erb
+++ b/app/views/packages/search.html.erb
@@ -3,15 +3,15 @@
<% if @packages.size > 0 %>
<div class="panel panel-default">
<div class="panel-heading">
- Results <%= @offset + 1 %>—<%= [@offset + Package.default_search_size, @packages.total].min %> of <%= @packages.total %>
+ Results <%= @offset + 1 %>—<%= [@offset + PackageRepository.default_search_size, @packages.total].min %> of <%= @packages.total %>
</div>
<div class="list-group">
<%= render partial: 'package_result_row', collection: @packages, as: 'package' %>
</div>
<div class="panel-footer">
<div class="btn-group" role="group" aria-label="Result navigation">
- <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - Package.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %>
- <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + Package.default_search_size), class: 'btn btn-default ' + ((@offset + Package.default_search_size) > @packages.total ? 'disabled' : '') %>
+ <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - PackageRepository.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %>
+ <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + PackageRepository.default_search_size), class: 'btn btn-default ' + ((@offset + PackageRepository.default_search_size) > @packages.total ? 'disabled' : '') %>
</div>
</div>
</div>
diff --git a/app/views/packages/stable.html.erb b/app/views/packages/stable.html.erb
index 7b230fe..d9654de 100644
--- a/app/views/packages/stable.html.erb
+++ b/app/views/packages/stable.html.erb
@@ -12,7 +12,7 @@
<% cache("stable-full-#{@changes.hash}") do %>
<ul class="list-group">
<% @changes.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %>
<% end %>
</ul>
diff --git a/app/views/packages/updated.html.erb b/app/views/packages/updated.html.erb
index b774c58..af54ce1 100644
--- a/app/views/packages/updated.html.erb
+++ b/app/views/packages/updated.html.erb
@@ -12,7 +12,7 @@
<% cache("updated-full-#{@changes.hash}") do %>
<ul class="list-group">
<% @changes.each do |change|
- _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %>
+ _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %>
<%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %>
<% end %>
</ul>
diff --git a/app/views/useflags/_useflag_result_row.html.erb b/app/views/useflags/_useflag_result_row.html.erb
index 084669f..3bdcd30 100644
--- a/app/views/useflags/_useflag_result_row.html.erb
+++ b/app/views/useflags/_useflag_result_row.html.erb
@@ -1,4 +1,4 @@
-<a class="list-group-item" href="<%= slf useflag_path useflag[:name] %>">
- <h3 class="kk-search-result-header"><%= useflag[:name] %></h3>
- <%= useflag[:description] %>
+<a class="list-group-item" href="<%= slf useflag_path useflag.name %>">
+ <h3 class="kk-search-result-header"><%= useflag.name %></h3>
+ <%= useflag.description %>
</a>
diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb
index 4ced5b5..1037b1f 100644
--- a/config/initializers/elasticsearch.rb
+++ b/config/initializers/elasticsearch.rb
@@ -1,9 +1,10 @@
-require 'elasticsearch/persistence/model'
+require 'elasticsearch/persistence'
+
+DEFAULT_CLIENT = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200'
-Elasticsearch::Persistence.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200'
if Rails.env.development? or ENV['RAILS_DEBUG']
logger = ActiveSupport::Logger.new(STDERR)
logger.level = Logger::DEBUG
logger.formatter = proc { |s, d, p, m| "\e[2m#{m}\n\e[0m" }
- Elasticsearch::Persistence.client.transport.logger = logger
+ DEFAULT_CLIENT.transport.logger = logger
end
diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb
index a1a2d93..a36f0ca 100644
--- a/lib/kkuleomi/store.rb
+++ b/lib/kkuleomi/store.rb
@@ -1,15 +1,12 @@
module Kkuleomi::Store
- def self.refresh_index
- Category.gateway.refresh_index!
- end
def self.create_index(force = false)
- types = [
- Category,
- Package,
- Version,
- Change,
- Useflag,
+ repositories = [
+ CategoryRepository,
+ PackageRepository,
+ VersionRepository,
+ ChangeRepository,
+ UseflagRepository,
]
base_settings = {
@@ -33,15 +30,11 @@ module Kkuleomi::Store
mapping: { total_fields: { limit: 50000 } }
}
+ settings = JSON.parse('{ "mapping": { "total_fields": { "limit": 50000 } } }')
+
# In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own.
- types.each { |type|
- client = type.gateway.client
- client.indices.delete(index: type.index_name) rescue nil if force
- body = {
- settings: type.settings.to_hash.merge(base_settings),
- mappings: type.mappings.to_hash
- }
- client.indices.create(index: type.index_name, body: body)
+ repositories.each { |repository|
+ repository.instance.create_index!(force: true, settings: settings)
}
end
end
diff --git a/lib/kkuleomi/store/model.rb b/lib/kkuleomi/store/model.rb
deleted file mode 100644
index 653884b..0000000
--- a/lib/kkuleomi/store/model.rb
+++ /dev/null
@@ -1,78 +0,0 @@
-module Kkuleomi::Store::Model
- def self.included(base)
- base.send :include, InstanceMethods
- base.extend ClassMethods
- end
-
- module ClassMethods
- # Finds instances by exact IDs using the 'term' filter
- def find_all_by(field, value, opts = {})
- search({
- size: 10_000,
- query: { bool: { filter: { term: { field => value } } } }
- }.merge(opts))
- end
-
- # Filter all instances by the given parameters
- def filter_all(filters, opts = {})
- filter_args = []
- filters.each_pair { |field, value| filter_args << { term: { field => value } } }
-
- search({
- query: {
- bool: { filter: { bool: { must: filter_args } } }
- },
- size: 10_000
- }.merge(opts))
- end
-
- def find_by(field, value, opts = {})
- find_all_by(field, value, opts).first
- end
-
- def find_all_by_parent(parent, opts = {})
- search(opts.merge(
- size: 10_000,
- query: {
- bool: {
- filter: {
- has_parent: {
- parent_type: parent.class.document_type,
- query: { term: { _id: parent.id } }
- }
- },
- must: { match_all: {} }
- }
- }
- ))
- end
-
- # Returns all (by default 10k) records of this class sorted by a field.
- def all_sorted_by(field, order, options = {})
- all({
- query: { match_all: {} },
- sort: { field => { order: order } }
- }, options)
- end
- end
-
- module InstanceMethods
- # Converts the model to an OpenStruct instance
- #
- # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil
- # @return [OpenStruct] OpenStruct containing the selected fields
- def to_os(*fields)
- fields = all_fields if fields.empty?
- OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }])
- end
-
- # Converts the model to a Hash
- #
- # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil
- # @return [Hash] Hash containing the selected fields
- def to_hsh(*fields)
- fields = all_fields if fields.empty?
- Hash[fields.map { |field| [field, send(field)] }]
- end
- end
-end
diff --git a/lib/kkuleomi/store/models/package_import.rb b/lib/kkuleomi/store/models/package_import.rb
index 99ab433..8ae1e5d 100644
--- a/lib/kkuleomi/store/models/package_import.rb
+++ b/lib/kkuleomi/store/models/package_import.rb
@@ -30,15 +30,15 @@ module Kkuleomi::Store::Models::PackageImport
set_basic_metadata(package_model, latest_ebuild)
# Be sure to have an ID now
- save
+ PackageRepository.save(self)
import_useflags!(package_model)
- Kkuleomi::Store.refresh_index
+ CategoryRepository.refresh_index!
import_versions!(package_model, ebuilds, options)
# Do this last, so that any exceptions before this point skip this step
self.metadata_hash = package_model.metadata_hash
- save
+ PackageRepository.save(self)
if options[:package_state] == 'new' && !options[:suppress_change_objects]
RecordChangeJob.perform_later(
@@ -73,7 +73,7 @@ module Kkuleomi::Store::Models::PackageImport
end
def import_useflags!(package_model)
- index_flags = Useflag.local_for(package_model.to_cp)
+ index_flags = UseflagRepository.local_for(package_model.to_cp)
model_flags = package_model.metadata[:use]
new_flags = model_flags.keys - index_flags.keys
@@ -87,23 +87,23 @@ module Kkuleomi::Store::Models::PackageImport
flag_doc.description = model_flags[flag]
flag_doc.atom = package_model.to_cp
flag_doc.scope = 'local'
- flag_doc.save
+ UseflagRepository.save(flag_doc)
end
eql_flags.each do |flag|
unless index_flags[flag].description == model_flags[flag]
index_flags[flag].description = model_flags[flag]
- index_flags[flag].save
+ UseflagRepository.save(index_flags[flag])
end
end
del_flags.each do |flag|
- index_flags[flag].delete
+ UseflagRepository.delete(index_flags[flag])
end
end
def import_versions!(package_model, ebuilds, options)
- index_v = Hash[Version.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }]
+ index_v = Hash[VersionRepository.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }]
model_v = Hash[ebuilds.map { |v| [v.version, v] }]
index_keys = index_v.keys
@@ -128,7 +128,7 @@ module Kkuleomi::Store::Models::PackageImport
if sort_key == 0
self.useflags = version_doc.useflags
- save
+ VersionRepository.save(version_doc)
end
end
@@ -144,12 +144,12 @@ module Kkuleomi::Store::Models::PackageImport
if sort_key == 0
self.useflags = version_doc.useflags
- save
+ VersionRepository.save(version_doc)
end
end
del_v.each do |v|
- index_v[v].delete
+ VersionRepository.delete(index_v[v])
end
end
end
diff --git a/lib/kkuleomi/store/models/package_search.rb b/lib/kkuleomi/store/models/package_search.rb
deleted file mode 100644
index ec0268c..0000000
--- a/lib/kkuleomi/store/models/package_search.rb
+++ /dev/null
@@ -1,161 +0,0 @@
-# Contains the search logic for packages
-module Kkuleomi::Store::Models::PackageSearch
- def self.included(base)
- base.send :include, InstanceMethods
- base.extend ClassMethods
- end
-
- module ClassMethods
- def suggest(q)
- Package.search(
- size: 20,
- query: {
- wildcard: {
- name_sort: {
- wildcard: q.downcase + '*'
- }
- }
- }
- )
- end
-
- # Tries to resolve a query atom to one or more packages
- def resolve(atom)
- [] if atom.nil? || atom.empty?
-
- Package.find_all_by(:atom, atom) + Package.find_all_by(:name, atom)
- end
-
- # Searches the versions index for versions using a certain USE flag.
- # Results are aggregated by package atoms.
- def find_atoms_by_useflag(useflag)
- Version.search(
- size: 0, # collect all packages.
- query: {
- bool: {
- must: { match_all: {} },
- filter: { term: { use: useflag } }
- }
- },
- aggs: {
- group_by_package: {
- terms: {
- field: 'package',
- order: { '_key' => 'asc' },
- # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html
- # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see.
- # Other limits in this app are also 10k mostly to 'make things fit kinda'.
- size: 10000,
- }
- }
- },
- ).response.aggregations['group_by_package'].buckets
- end
-
- def default_search_size
- 25
- end
-
- def default_search(q, offset)
- return [] if q.nil? || q.empty?
-
- part1, part2 = q.split('/', 2)
-
- if part2.nil?
- search(build_query(part1, nil, default_search_size, offset))
- else
- search(build_query(part2, part1, default_search_size, offset))
- end
- end
-
- def build_query(q, category, size, offset)
- {
- size: size,
- from: offset,
- query: {
- function_score: {
- query: { bool: bool_query_parts(q, category) },
- functions: scoring_functions
- }
- }
- }
- end
-
- def bool_query_parts(q, category = nil)
- q_dwncsd = q.downcase
-
- query = {
- must: [
- match_wildcard(q_dwncsd)
- ],
- should: [
- match_phrase(q_dwncsd),
- match_description(q)
- ]
- }
-
- query[:must] << [match_category(category)] if category
-
- query
- end
-
- def match_wildcard(q)
- q = ('*' + q + '*') unless q.include? '*'
- q.tr!(' ', '*')
-
- {
- wildcard: {
- name_sort: {
- wildcard: q,
- boost: 4
- }
- }
- }
- end
-
- def match_phrase(q)
- {
- match_phrase: {
- name: {
- query: q,
- boost: 5
- }
- }
- }
- end
-
- def match_description(q)
- {
- match: {
- description: {
- query: q,
- boost: 0.1
- }
- }
- }
- end
-
- def match_category(cat)
- {
- match: {
- category: {
- query: cat,
- boost: 2
- }
- }
- }
- end
-
- def scoring_functions
- [
- {
- filter: { term: { category: 'virtual' } },
- weight: 0.6
- }
- ]
- end
- end
-
- module InstanceMethods
- end
-end
diff --git a/lib/kkuleomi/store/models/version_import.rb b/lib/kkuleomi/store/models/version_import.rb
index b65b683..6ee6b64 100644
--- a/lib/kkuleomi/store/models/version_import.rb
+++ b/lib/kkuleomi/store/models/version_import.rb
@@ -38,7 +38,7 @@ module Kkuleomi::Store::Models::VersionImport
self.masks = Portage::Util::Masks.for(ebuild_model)
self.metadata_hash = ebuild_model.metadata_hash
- save()
+ VersionRepository.save(self)
# If keywords changed, calculate changes and record as needed (but only do that if we should)
unless options[:suppress_change_objects]
@@ -60,7 +60,7 @@ module Kkuleomi::Store::Models::VersionImport
# @param [Package] parent Parent package model
def set_sort_key!(key, parent)
self.sort_key = key
- save()
+ VersionRepository.save(self)
end
def strip_useflag_defaults(flags)