68 lines
2.0 KiB
Ruby
68 lines
2.0 KiB
Ruby
#!/usr/bin/env ruby
|
|
ENV['RAILS_ENV'] = 'production'
|
|
require File.dirname(__FILE__) + '/../config/environment'
|
|
include FullTextSearch
|
|
|
|
#TODO duplicate code in FullTextSearch
|
|
def build_query_terms(query)
|
|
pq = Hash.new
|
|
fields = [:title, :body]
|
|
|
|
Topic.search_with_index(Topic) do |index|
|
|
reader = index.reader
|
|
term_freq_map = Hash.new(0)
|
|
index.search_each(query, :sort => "created_at DESC", :limit => :all) {|id, score|
|
|
fields.each do |field|
|
|
term_freq_vector = reader.term_vector(id, field)
|
|
term_freq_vector.terms.each { |term| term_freq_map[term.text] += term.positions.size } if term_freq_vector
|
|
end
|
|
}
|
|
|
|
num_docs = reader.num_docs
|
|
|
|
|
|
term_freq_map.each_pair do |word, tf|
|
|
# filter out words that don't occur enough times in the source
|
|
next if tf < 2
|
|
# go through all the fields and find the largest document frequency
|
|
doc_freq = 0
|
|
fields.each do |field_name|
|
|
freq = reader.doc_freq(field_name, word)
|
|
if freq > doc_freq
|
|
doc_freq = freq
|
|
end
|
|
end
|
|
|
|
# filter out words that don't occur in enough docs
|
|
next if doc_freq < 5
|
|
next if doc_freq == 0 # index update problem ?
|
|
|
|
idf = (num_docs == 0) ? 0.0 : Math.log(num_docs.to_f/doc_freq)
|
|
pq[word] = tf * idf
|
|
end
|
|
end
|
|
|
|
return pq
|
|
end
|
|
|
|
queries = Hash.new
|
|
queries["java"] = build_query_terms("forum_id:39")
|
|
queries["ruby"] = build_query_terms("forum_id:40")
|
|
queries["ajax"] = build_query_terms("forum_id:41")
|
|
queries["agile"] = build_query_terms("forum_id:42")
|
|
queries["job"] = build_query_terms("forum_id:44")
|
|
queries["life"] = build_query_terms("forum_id:45")
|
|
|
|
types = ["java", "ruby", "ajax", "agile", "job", "life"]
|
|
|
|
types.each { |type|
|
|
pq = queries[type].reject { |word, score|
|
|
score < queries.values.map{ |q| q[word] }.compact.max
|
|
}.sort {|a, b|
|
|
a[1] <=> b[1]
|
|
}[-500,500]
|
|
|
|
c = CachedContent.find_or_initialize_by_content_key(type)
|
|
c.content = pq
|
|
c.save!
|
|
} |