263 lines
11 KiB
Ruby
263 lines
11 KiB
Ruby
#!/usr/bin/env ruby
|
||
# -*- coding: utf-8 -*-
|
||
|
||
ENV['RAILS_ENV'] ||= 'production'
|
||
require File.dirname(__FILE__) + '/../config/environment'
|
||
require 'open-uri'
|
||
require 'pathname'
|
||
|
||
frequency = ARGV[0] || 'daily'
|
||
MAX_SIZE = 50000
|
||
SITEMAP_FILE_PATH = "#{RAILS_ROOT}/public/sitemaps/"
|
||
SITEMAP_URL_PATH = "http://www.#{DEFAULT_ROOT_DOMAIN}/sitemaps/"
|
||
|
||
# config each channel infos to generate sitemap
|
||
# [:name] 区分不同的sitemap，也是对应sitemap文件的一部分
|
||
# [:channel_class] 数据的model class, 使用其find（find_in_batches）方法来查询数据
|
||
# [:query_options] 查询参数,daily的查询条件会据此结合:daily_field来形成自己的查询条件
|
||
# [:init_proc] 准备sitemap文件生成所需的数据格式定义：url、更新时间
|
||
channels =
|
||
[{ :name => "topics", :channel_class => Topic,
|
||
:query_options => { :conditions => ["topics.forum_id IS NOT NULL AND topics.status_flag <> 'hidden' AND topics.status_flag <> 'delete'"], :joins => "join posts on topics.last_post_id = posts.id" },
|
||
:daily_field => "topics.created_at",
|
||
:init_proc => Proc.new{ Topic.class_eval do
|
||
def loc
|
||
format 'http://www.%s/topic/%s', DEFAULT_ROOT_DOMAIN, self.id
|
||
end
|
||
def lastmod
|
||
(self.last_post || self).created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "blogs", :channel_class => Blog,
|
||
:query_options => { :conditions => ['blogs.status <> ? and blogs.status <> ?', Blog.status[:draft], Blog.status[:deleted]], :joins => "join users on blogs.user_id = users.id" },
|
||
:daily_field => "blogs.created_at",
|
||
:init_proc => Proc.new{ Blog.class_eval do
|
||
def loc
|
||
format '%sblog/%s', self.user.homepage, self.id
|
||
end
|
||
def lastmod
|
||
created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "problems", :channel_class => Problem,
|
||
:query_options => { :conditions => ["1=1"]},
|
||
:daily_field => "created_at",
|
||
:init_proc => Proc.new{ Problem.class_eval do
|
||
def loc
|
||
format 'http://www.%s/problems/%s', DEFAULT_ROOT_DOMAIN, self.id
|
||
end
|
||
def lastmod
|
||
source = self.solutions.max_by{|solution| solution.created_at } || self
|
||
source.created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "news", :channel_class => News,
|
||
:query_options => { :conditions => ["status = 'approved'"]},
|
||
:daily_field => "created_at",
|
||
:init_proc => Proc.new{ News.class_eval do
|
||
def loc
|
||
format 'http://www.%s/news/%s-%s', DEFAULT_ROOT_DOMAIN, self.id, URI.escape(self.slug_url.to_s)
|
||
end
|
||
def lastmod
|
||
created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "magazines", :channel_class => Essence,
|
||
:query_options => { :conditions => ["status = 'approved'"]},
|
||
:daily_field => "created_at",
|
||
:init_proc => Proc.new{ Essence.class_eval do
|
||
def loc
|
||
format 'http://www.%s/magazines/%s-%s', DEFAULT_ROOT_DOMAIN, self.id, URI.escape(self.slug_url.to_s)
|
||
end
|
||
def lastmod
|
||
created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "group", :channel_class => Group,
|
||
:query_options => { :conditions => ["active = ? and id not in (?)", true, Group::HIDDEN_GROUP_IDS]},
|
||
:daily_field => "created_at",
|
||
:init_proc => Proc.new{ Group.class_eval do
|
||
def loc
|
||
format 'http://%s.group.%s', URI.escape(self.domain), DEFAULT_ROOT_DOMAIN
|
||
end
|
||
def lastmod
|
||
created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "group_topic", :channel_class => GroupTopic,
|
||
:query_options => { :conditions => ["1=1"], :joins => "join group_posts on group_topics.last_post_id = group_posts.id left join groups on groups.id=group_topics.group_id"},
|
||
:daily_field => "group_topics.created_at",
|
||
:init_proc => Proc.new{ GroupTopic.class_eval do
|
||
def loc
|
||
format 'http://%s.group.%s/group/topic/%s', URI.escape(self.group.domain), DEFAULT_ROOT_DOMAIN, self.id
|
||
end
|
||
def lastmod
|
||
created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "group_wiki", :channel_class => Page,
|
||
:query_options => { :conditions => ["pages.group_id is not null and pages.group_id not in (?) and pages.status <> 'delete'", Group::HIDDEN_GROUP_IDS], :joins => "join groups on groups.id=pages.group_id"},
|
||
:daily_field => "pages.created_at",
|
||
:init_proc => Proc.new{ Page.class_eval do
|
||
def loc
|
||
format 'http://%s.group.%s%s', URI.escape(self.group.domain), DEFAULT_ROOT_DOMAIN, URI.escape(self.slug_url)
|
||
end
|
||
def lastmod
|
||
created_at.xmlschema
|
||
end
|
||
end}},
|
||
{ :name => "magazine_type", :channel_class => Essence,
|
||
:query_options => { :select => "essence_type as id, max(created_at) as created_at", :conditions => ["1=1"], :group => "essence_type"},
|
||
:init_proc => Proc.new{ Essence.class_eval do
|
||
def loc
|
||
format 'http://www.%s/magazines/%s', DEFAULT_ROOT_DOMAIN, Essence::ESSENCE_TYPE[self.id].try(:last)
|
||
end
|
||
end}},
|
||
{ :name => "problem_category", :channel_class => Forum,
|
||
:query_options => { :conditions => ["1=1"]},
|
||
:init_proc => Proc.new{ Forum.class_eval do
|
||
def loc
|
||
format 'http://www.%s/problems/tags/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.urlname)
|
||
end
|
||
end}},
|
||
{ :name => "whole_tag", :channel_class => Whole::Tag,
|
||
:query_options => { :select => "id, name", :conditions => "category_id > 0" },
|
||
:init_proc => Proc.new { Whole::Tag.class_eval do
|
||
def loc
|
||
[format('http://www.%s/news/tag/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.name)),
|
||
format('http://www.%s/magazines/tag/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.name)),
|
||
format('http://www.%s/blogs/tag/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.name)) ]
|
||
end
|
||
end }},
|
||
{ :name => "whole_category", :channel_class => WholeCategory,
|
||
:query_options => { :select => "id, urlname" },
|
||
:init_proc => Proc.new { WholeCategory.class_eval do
|
||
def loc
|
||
[format('http://www.%s/news/category/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.urlname)),
|
||
format('http://www.%s/magazines/category/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.urlname)),
|
||
format('http://www.%s/blogs/category/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.urlname)) ]
|
||
end
|
||
end }},
|
||
{ :name => "problem_tag", :channel_class => SysTag,
|
||
:query_options => { :select => "sys_tags.id, sys_tags.urlname", :conditions => "sys_tags.forum_id is not null", :joins => "join problems on sys_tags.id = problems.sys_tag_id", :group => "sys_tags.id" },
|
||
:init_proc => Proc.new{ SysTag.class_eval do
|
||
def loc
|
||
format 'http://www.%s/problems/tags/%s', DEFAULT_ROOT_DOMAIN, URI.escape(self.urlname)
|
||
end
|
||
end}}
|
||
]
|
||
|
||
# 跳过生成daily范围的网站地图
|
||
without_daily_channels = ["whole_category", "whole_tag", "magazine_type", "problem_category"]
|
||
# 需要二级域名访问的地址
|
||
subdomain_channels = %w(blogs group group_topic group_wiki)
|
||
|
||
module Enumerable
|
||
|
||
def to_sitemap
|
||
builder = Builder::XmlMarkup.new(:indent => 2)
|
||
builder.instruct!
|
||
sitemap = builder.urlset :xmlns => 'http://www.sitemaps.org/schemas/sitemap/0.9' do
|
||
each do |item|
|
||
if item.loc.is_a? Array
|
||
item.loc.each do |loc|
|
||
builder.url do
|
||
builder.loc loc
|
||
end
|
||
end
|
||
else
|
||
builder.url do
|
||
builder.loc item.loc
|
||
builder.lastmod item.lastmod if defined?(item.lastmod)
|
||
end
|
||
end
|
||
end
|
||
end
|
||
sitemap
|
||
end
|
||
|
||
def write_sitemap(prefix, last = false)
|
||
FileUtils.mkdir_p(SITEMAP_FILE_PATH)
|
||
unless empty?
|
||
file_name = last ? "#{SITEMAP_FILE_PATH}#{prefix}_last.xml" : "#{SITEMAP_FILE_PATH}#{prefix}_#{first.id}.xml"
|
||
# puts "====== writing sitemap file : #{file_name}..."
|
||
File.open(file_name, 'w'){ |f| f.write to_sitemap}
|
||
# puts "====== writed sitemap file : #{file_name}"
|
||
end
|
||
end
|
||
|
||
end
|
||
|
||
def sitemaps
|
||
Pathname.new(SITEMAP_FILE_PATH).children
|
||
end
|
||
|
||
def gzip_sitemaps
|
||
`gzip #{SITEMAP_FILE_PATH}*.xml`
|
||
`rm #{SITEMAP_FILE_PATH}sitemap_index.xml.gz` # 索引文件不压缩
|
||
end
|
||
|
||
def clean_sitemaps
|
||
sitemaps.each {|s| s.delete unless s.to_s =~ /last/}
|
||
end
|
||
|
||
def clean_daily_sitemaps
|
||
sitemaps.each {|s| s.delete if s.to_s =~ /last/}
|
||
end
|
||
|
||
def generate_sitemap_index
|
||
File.open("#{SITEMAP_FILE_PATH}sitemap_index.xml", 'w') do |f|
|
||
builder = Builder::XmlMarkup.new(:indent => 2)
|
||
builder.instruct!
|
||
sitemap = builder.sitemapindex :xmlns => 'http://www.sitemaps.org/schemas/sitemap/0.9' do
|
||
sitemaps.each do |file|
|
||
if ['.xml','.gz'].include?(file.extname) && file.basename.to_s != 'sitemap_index.xml'
|
||
builder.sitemap do
|
||
builder.loc "#{SITEMAP_URL_PATH}#{file.basename}"
|
||
end
|
||
end
|
||
end
|
||
end
|
||
|
||
f.write sitemap
|
||
end
|
||
end
|
||
|
||
case frequency
|
||
when 'weekly'
|
||
clean_sitemaps
|
||
puts "== generating sitemaps weekly: #{Time.now()}..."
|
||
puts " channels: #{(channels.collect{|e| e[:name]} - subdomain_channels).join(', ')}"
|
||
channels.each do |channel|
|
||
next if (subdomain_channels).include?(channel[:name])
|
||
channel[:init_proc].call
|
||
channel[:channel_class].find_in_batches({ :batch_size => MAX_SIZE}.merge(channel[:query_options])) do |records|
|
||
puts "==== generating sitemap #{channel[:name]} (weekly) : record count(#{records.count}) ,#{Time.now.to_s(:db)}..."
|
||
records.write_sitemap(channel[:name])
|
||
puts " generated sitemap #{channel[:name]} (weekly) : record count(#{records.count}) ,#{Time.now.to_s(:db)}"
|
||
end
|
||
end
|
||
puts "== generated sitemaps weekly: #{Time.now.to_s(:db)}"
|
||
when 'daily'
|
||
clean_daily_sitemaps
|
||
puts "== generating sitemaps daily: #{Time.now()}"
|
||
puts " channels: #{(channels.collect{|e| e[:name]} - subdomain_channels - without_daily_channels).join(', ')}"
|
||
channels.each do |channel|
|
||
next if (subdomain_channels + without_daily_channels).include?(channel[:name])
|
||
channel[:init_proc].call
|
||
daily_conditions = channel[:query_options][:conditions]
|
||
daily_conditions[0] << " and #{channel[:daily_field]} > ? " and daily_conditions << Time.now.beginning_of_week if channel[:daily_field]
|
||
records = channel[:channel_class].all(channel[:query_options].merge({ :conditions => daily_conditions}))
|
||
puts "==== generating sitemap #{channel[:name]} (daily) : record count(#{records.count}) , #{Time.now.to_s(:db)}..."
|
||
records.write_sitemap(channel[:name], true)
|
||
puts " generated sitemap #{channel[:name]} (daily) : record count(#{records.count}) , #{Time.now.to_s(:db)}"
|
||
end
|
||
puts "== generated sitemaps daily: #{Time.now.to_s(:db)}"
|
||
else
|
||
puts 'xxxx Unknown update frequency , use \'daily\'(default) or \'weekly\'.'
|
||
end
|
||
|
||
gzip_sitemaps
|
||
|
||
generate_sitemap_index
|