Initial commit w/currently running code

9 files changed, 657 insertions, 0 deletions

diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb
new file mode 100644
index 0000000..4baf2c4
--- /dev/null
+++ b/lib/kkuleomi/store.rb
@@ -0,0 +1,53 @@
+module Kkuleomi::Store
+  def self.refresh_index
+    Category.gateway.refresh_index!
+  end
+
+  def self.create_index(force = false)
+    client = Category.gateway.client
+    index_name = Category.index_name
+
+    settings_list = [
+      Category.settings.to_hash,
+      Package.settings.to_hash,
+      Version.settings.to_hash,
+      Change.settings.to_hash,
+      Useflag.settings.to_hash
+    ]
+
+    mappings_list = [
+      Category.mappings.to_hash,
+      Package.mappings.to_hash,
+      Version.mappings.to_hash,
+      Change.mappings.to_hash,
+      Useflag.mappings.to_hash
+    ]
+
+    settings = {
+        analysis: {
+            filter: {
+                autocomplete_filter: {
+                    type: 'edge_ngram',
+                    min_gram: 1,
+                    max_gram: 20,
+                }
+            },
+            analyzer: {
+                autocomplete: {
+                    type: 'custom',
+                    tokenizer: 'standard',
+                    filter: %w(lowercase autocomplete_filter)
+                }
+            }
+        }
+    }
+    settings_list.each { |setting| settings.merge! setting }
+
+    mappings = {}
+    mappings_list.each { |mapping| mappings.merge! mapping }
+
+    client.indices.delete(index: index_name) rescue nil if force
+
+    client.indices.create(index: index_name, body: { settings: settings, mappings: mappings })
+  end
+end
diff --git a/lib/kkuleomi/store/model.rb b/lib/kkuleomi/store/model.rb
new file mode 100644
index 0000000..8d406bf
--- /dev/null
+++ b/lib/kkuleomi/store/model.rb
@@ -0,0 +1,80 @@
+module Kkuleomi::Store::Model
+  def self.included(base)
+    base.send :include, InstanceMethods
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    # Finds instances by exact IDs using the 'term' filter
+    def find_all_by(field, value, opts = {})
+      search({
+        size: 10_000,
+        query: {
+          filtered: { filter: { term: { field => value } } }
+        }
+      }.merge(opts))
+    end
+
+    # Filter all instances by the given parameters
+    def filter_all(filters, opts = {})
+      filter_args = []
+      filters.each_pair { |field, value| filter_args << { term: { field => value } } }
+
+      search({
+        query: {
+          filtered: { filter: { bool: { must: filter_args } } }
+        },
+        size: 10_000
+      }.merge(opts))
+    end
+
+    def find_by(field, value, opts = {})
+      find_all_by(field, value, opts).first
+    end
+
+    def find_all_by_parent(parent, opts = {})
+      search(opts.merge(
+        size: 10_000,
+        query: {
+          filtered: {
+            filter: {
+              has_parent: {
+                type: parent.class.document_type,
+                filter: { term: { _id: parent.id } }
+              }
+            },
+            query: { match_all: {} }
+          }
+        }
+      ))
+    end
+
+    # Returns all (by default 10k) records of this class sorted by a field.
+    def all_sorted_by(field, order, options = {})
+      all({
+            query: { match_all: {} },
+            sort: { field => { order: order } }
+          }, options)
+    end
+  end
+
+  module InstanceMethods
+    # Converts the model to an OpenStruct instance
+    #
+    # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil
+    # @return [OpenStruct] OpenStruct containing the selected fields
+    def to_os(*fields)
+      fields = all_fields if fields.empty?
+      OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }])
+    end
+
+    # Converts the model to a Hash
+    #
+    # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil
+    # @return [Hash] Hash containing the selected fields
+    def to_hsh(*fields)
+      fields = all_fields if fields.empty?
+      Hash[fields.map { |field| [field, send(field)] }]
+    end
+  end
+end
diff --git a/lib/kkuleomi/store/models.rb b/lib/kkuleomi/store/models.rb
new file mode 100644
index 0000000..6b0c4e3
--- /dev/null
+++ b/lib/kkuleomi/store/models.rb
@@ -0,0 +1,2 @@
+module Kkuleomi::Store::Models
+end
diff --git a/lib/kkuleomi/store/models/package_import.rb b/lib/kkuleomi/store/models/package_import.rb
new file mode 100644
index 0000000..caea415
--- /dev/null
+++ b/lib/kkuleomi/store/models/package_import.rb
@@ -0,0 +1,154 @@
+# Contains the import logic for packages
+module Kkuleomi::Store::Models::PackageImport
+  def self.included(base)
+    base.send :include, InstanceMethods
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+  end
+
+  module InstanceMethods
+    # Determines if the current package document needs an update from the model
+    #
+    # @param [Portage::Repository::Package] package_model Package model
+    def needs_import?(package_model)
+      metadata_hash != package_model.metadata_hash
+    end
+
+    # Imports data from this model. Saving is not optional here as we might need the package's ID as parent.
+    #
+    # @param [Portage::Repository::Package] package_model Package model
+    # @param [Hash] options Import options
+    def import!(package_model, options)
+      # Fetch ebuilds, newest-first
+      ebuilds = package_model.ebuilds_sorted.reverse
+      latest_ebuild = ebuilds.first
+
+      fail "No ebuilds found for #{package_model.name}. Skipping import." unless latest_ebuild
+
+      set_basic_metadata(package_model, latest_ebuild)
+
+      # Be sure to have an ID now
+      save
+
+      import_useflags!(package_model)
+      Kkuleomi::Store.refresh_index
+      import_versions!(package_model, ebuilds, options)
+
+      # Do this last, so that any exceptions before this point skip this step
+      self.metadata_hash = package_model.metadata_hash
+      save
+
+      if options[:package_state] == 'new' && !options[:suppress_change_objects]
+        RecordChangeJob.perform_later(
+          type: 'new_package',
+          category: category,
+          package: name
+        )
+      end
+    end
+
+    def set_basic_metadata(package_model, latest_ebuild)
+      self.name = package_model.name
+      self.name_sort = package_model.name.downcase
+      self.category = package_model.category
+      self.atom = package_model.to_cp
+
+      self.description = latest_ebuild.metadata[:description]
+
+      if (homepage = latest_ebuild.metadata[:homepage])
+        self.homepage = homepage.split ' '
+      end
+
+      self.license = latest_ebuild.metadata[:license]
+      self.licenses = split_license_str latest_ebuild.metadata[:license]
+
+      self.herds = package_model.metadata[:herds]
+      self.maintainers = package_model.metadata[:maintainer]
+
+      self.longdescription = package_model.metadata[:longdescription][:en]
+    end
+
+    def import_useflags!(package_model)
+      index_flags = Useflag.local_for(package_model.to_cp)
+      model_flags = package_model.metadata[:use]
+
+      new_flags = model_flags.keys - index_flags.keys
+      del_flags = index_flags.keys - model_flags.keys
+      eql_flags = model_flags.keys & index_flags.keys
+
+      new_flags.each do |flag|
+        flag_doc = Useflag.new
+        # TODO: import! method?
+        flag_doc.name = flag
+        flag_doc.description = model_flags[flag]
+        flag_doc.atom = package_model.to_cp
+        flag_doc.scope = 'local'
+        flag_doc.save
+      end
+
+      eql_flags.each do |flag|
+        unless index_flags[flag].description == model_flags[flag]
+          index_flags[flag].description = model_flags[flag]
+          index_flags[flag].save
+        end
+      end
+
+      del_flags.each do |flag|
+        index_flags[flag].delete
+      end
+    end
+
+    def import_versions!(package_model, ebuilds, options)
+      index_v = Hash[Version.find_all_by_parent(self).map { |v| [v.version, v] }]
+      model_v = Hash[ebuilds.map { |v| [v.version, v] }]
+
+      index_keys = index_v.keys
+      model_keys = model_v.keys
+
+      new_v = model_keys - index_keys
+      del_v = index_keys - model_keys
+      eql_v = model_keys & index_keys
+
+      Rails.logger.debug { "#{package_model.to_cp} new: " + new_v.inspect }
+      Rails.logger.debug { "#{package_model.to_cp} del: " + del_v.inspect }
+      Rails.logger.debug { "#{package_model.to_cp} eql: " + eql_v.inspect }
+
+      ebuild_order = Hash[ebuilds.each_with_index.map { |e, i| [e.version, i] }]
+
+      new_v.each do |v|
+        version_doc = Version.new
+        version_doc.import!(model_v[v], self, options.merge(version_state: 'new'))
+
+        sort_key = ebuild_order[v]
+        version_doc.set_sort_key!(sort_key, self)
+
+        if sort_key == 0
+          self.useflags = version_doc.useflags
+          save
+        end
+      end
+
+      eql_v.each do |v|
+        version_doc = index_v[v]
+
+        if version_doc.needs_import? model_v[v]
+          version_doc.import!(model_v[v], self, options)
+        end
+
+        sort_key = ebuild_order[v]
+        version_doc.set_sort_key!(sort_key, self)
+
+        if sort_key == 0
+          self.useflags = version_doc.useflags
+          save
+        end
+      end
+
+      del_v.each do |v|
+        index_v[v].delete
+      end
+    end
+  end
+end
diff --git a/lib/kkuleomi/store/models/package_search.rb b/lib/kkuleomi/store/models/package_search.rb
new file mode 100644
index 0000000..150b0e9
--- /dev/null
+++ b/lib/kkuleomi/store/models/package_search.rb
@@ -0,0 +1,158 @@
+# Contains the search logic for packages
+module Kkuleomi::Store::Models::PackageSearch
+  def self.included(base)
+    base.send :include, InstanceMethods
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    def suggest(q)
+      Package.search(
+        size: 20,
+        query: {
+          wildcard: {
+            name_sort: {
+              wildcard: q.downcase + '*'
+            }
+          }
+        }
+      )
+    end
+
+    # Tries to resolve a query atom to one or more packages
+    def resolve(atom)
+      [] if atom.nil? || atom.empty?
+
+      Package.find_all_by(:atom, atom) + Package.find_all_by(:name, atom)
+    end
+
+    # Searches the versions index for versions using a certain USE flag.
+    # Results are aggregated by package atoms.
+    def find_atoms_by_useflag(useflag)
+      Version.search(
+        query: {
+          filtered: {
+            query: { match_all: {} },
+            filter: { term: { use: useflag } }
+          }
+        },
+        aggs: {
+          group_by_package: {
+            terms: {
+              field: 'package',
+              size: 0,
+              order: { '_term' => 'asc' }
+            }
+          }
+        },
+        size: 0
+      ).response.aggregations['group_by_package'].buckets
+    end
+
+    def default_search_size
+      25
+    end
+
+    def default_search(q, offset)
+      return [] if q.nil? || q.empty?
+
+      part1, part2 = q.split('/', 2)
+
+      if part2.nil?
+        search(build_query(part1, nil, default_search_size, offset))
+      else
+        search(build_query(part2, part1, default_search_size, offset))
+      end
+    end
+
+    def build_query(q, category, size, offset)
+      {
+        size: size,
+        from: offset,
+        query: {
+          function_score: {
+            query: { bool: bool_query_parts(q, category) },
+            functions: scoring_functions
+          }
+        }
+      }
+    end
+
+    def bool_query_parts(q, category = nil)
+      q_dwncsd = q.downcase
+
+      query = {
+        must: [
+          match_wildcard(q_dwncsd)
+        ],
+        should: [
+          match_phrase(q_dwncsd),
+          match_description(q)
+        ]
+      }
+
+      query[:must] << [match_category(category)] if category
+
+      query
+    end
+
+    def match_wildcard(q)
+      q = ('*' + q + '*') unless q.include? '*'
+      q.tr!(' ', '*')
+
+      {
+        wildcard: {
+          name_sort: {
+            wildcard: q,
+            boost: 4
+          }
+        }
+      }
+    end
+
+    def match_phrase(q)
+      {
+        match_phrase: {
+          name: {
+            query: q,
+            boost: 5
+          }
+        }
+      }
+    end
+
+    def match_description(q)
+      {
+        match: {
+          description: {
+            query: q,
+            boost: 0.1
+          }
+        }
+      }
+    end
+
+    def match_category(cat)
+      {
+        match: {
+          category: {
+            query: cat,
+            boost: 2
+          }
+        }
+      }
+    end
+
+    def scoring_functions
+      [
+        {
+          filter: { term: { category: 'virtual' } },
+          boost_factor: 0.6
+        }
+      ]
+    end
+  end
+
+  module InstanceMethods
+  end
+end
diff --git a/lib/kkuleomi/store/models/version_import.rb b/lib/kkuleomi/store/models/version_import.rb
new file mode 100644
index 0000000..c18344e
--- /dev/null
+++ b/lib/kkuleomi/store/models/version_import.rb
@@ -0,0 +1,120 @@
+# Contains the import logic for versions
+module Kkuleomi::Store::Models::VersionImport
+  def self.included(base)
+    base.send :include, InstanceMethods
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+  end
+
+  module InstanceMethods
+    # Determines if the current version document needs an update from the model
+    #
+    # @param [Portage::Repository::Ebuild] ebuild_model Ebuild model
+    def needs_import?(ebuild_model)
+      metadata_hash != ebuild_model.metadata_hash
+    end
+
+    # Imports data from an ebuild model and saves the object
+    #
+    # @param [Portrage::Repository::Ebuild] ebuild_model
+    def import!(ebuild_model, parent_package, options)
+      self.version = ebuild_model.version
+      self.atom = ebuild_model.to_cpv
+      self.package = parent_package.atom
+
+      raw_slot = nil
+      raw_subslot = nil
+      raw_slot, raw_subslot = ebuild_model.metadata[:slot].split '/' if ebuild_model.metadata[:slot]
+      self.slot = raw_slot || ''
+      self.subslot = raw_subslot || ''
+
+      old_keywords = keywords
+      self.keywords = ebuild_model.metadata[:keywords] || []
+      self.use = strip_useflag_defaults(ebuild_model.metadata[:iuse] || []).uniq
+      self.restrict = ebuild_model.metadata[:restrict] || []
+      self.properties = ebuild_model.metadata[:properties] || []
+      self.masks = Portage::Util::Masks.for(ebuild_model)
+      self.metadata_hash = ebuild_model.metadata_hash
+
+      save(parent: parent_package.id)
+
+      # If keywords changed, calculate changes and record as needed (but only do that if we should)
+      unless options[:suppress_change_objects]
+        RecordChangeJob.perform_later(
+          type: 'version_bump',
+          category: parent_package.category,
+          package: parent_package.name,
+          version: version
+        ) if options[:package_state] != 'new' && options[:version_state] == 'new'
+
+        process_keyword_diff(old_keywords, keywords, parent_package) unless old_keywords == keywords
+      end
+    end
+
+    # Convenience method to set the sort key and save the model
+    #
+    # @param [Integer] sort_key Sort key to set
+    # @param [Package] parent Parent package model
+    def set_sort_key!(key, parent)
+      self.sort_key = key
+      save(parent: parent.id)
+    end
+
+    def strip_useflag_defaults(flags)
+      flags.map { |flag| flag.start_with?('+', '-') ? flag[1..-1] : flag }
+    end
+
+    def process_keyword_diff(old_kws_raw, new_kws_raw, package)
+      stabled = []
+      keyworded = []
+
+      old_kws = parse_keywords old_kws_raw
+      new_kws = parse_keywords new_kws_raw
+
+      (old_kws[:arches].keys | new_kws[:arches].keys).each do |arch|
+        old = old_kws[:arches][arch]
+        new = new_kws[:arches][arch]
+
+        if old && new
+          next if old == new
+
+          if old == :unavailable && new == :testing
+            keyworded << arch
+          elsif old == :unavailable && new == :stable
+            stabled << arch
+          elsif old == :testing && new == :stable
+            stabled << arch
+          end
+        elsif new && !old
+          if new == :testing
+            keyworded << arch
+          elsif new == :stable
+            stabled << arch
+          end
+        end
+      end
+
+      unless stabled.empty?
+        RecordChangeJob.perform_later(
+          type: 'stable',
+          category: package.category,
+          package: package.name,
+          version: version,
+          arches: stabled
+        )
+      end
+
+      unless keyworded.empty?
+        RecordChangeJob.perform_later(
+          type: 'keyword',
+          category: package.category,
+          package: package.name,
+          version: version,
+          arches: keyworded
+        )
+      end
+    end
+  end
+end
diff --git a/lib/kkuleomi/store/suggester.rb b/lib/kkuleomi/store/suggester.rb
new file mode 100644
index 0000000..1da59ab
--- /dev/null
+++ b/lib/kkuleomi/store/suggester.rb
@@ -0,0 +1,23 @@
+class Kkuleomi::Store::Suggester
+  def initialize(q)
+    @q = q
+  end
+
+  def response
+    @response ||= begin
+      Elasticsearch::Persistence.client.suggest(
+          index: "packages-#{Rails.env}",
+          body: {
+              name: {
+                  text: @q,
+                  completion: { field: 'suggest_name', size: 25 }
+              },
+              description: {
+                  text: @q,
+                  completion: { field: 'suggest_description', size: 25 }
+              }
+          }
+      )
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/kkuleomi/util.rb b/lib/kkuleomi/util.rb
new file mode 100644
index 0000000..e9ea9be
--- /dev/null
+++ b/lib/kkuleomi/util.rb
@@ -0,0 +1,2 @@
+module Kkuleomi::Util
+end
diff --git a/lib/kkuleomi/util/exec.rb b/lib/kkuleomi/util/exec.rb
new file mode 100644
index 0000000..704d651
--- /dev/null
+++ b/lib/kkuleomi/util/exec.rb
@@ -0,0 +1,65 @@
+require 'open3'
+
+class Kkuleomi::Util::Exec
+  def initialize(command)
+    @cmd = command
+  end
+
+  def env(key, value)
+    @env ||= {}
+    @env[key] = value
+
+    self
+  end
+
+  def args(*arguments)
+    @args = arguments
+
+    self
+  end
+
+  def in(chdir)
+    @chdir = chdir
+
+    self
+  end
+
+  def run
+    args = nil
+
+    if @env.is_a?(Hash) && !@env.empty?
+      args = [@env, @cmd]
+    else
+      args = @cmd
+    end
+
+    opts = {}
+    opts[:chdir] = @chdir if @chdir
+
+    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(args, *(@args || []), opts)
+
+    @ran = true
+    self
+  end
+
+  def exit_status
+    run unless @ran
+    @wait_thr.value
+  end
+
+  def stdout
+    run unless @ran
+    @stdout.read
+  end
+
+  def stderr
+    run unless @ran
+    @stderr.read
+  end
+
+  class << self
+    def cmd(command)
+      new(command)
+    end
+  end
+end