edshot99/eBooru
1
0
Fork 0
forked from e621ng/e621ng
Code Issues 11 Releases 1 Activity

[Cleanup] Remove sftp storage manager

Browse Source
This commit is contained in:
Earlopain 2022-04-07 18:23:02 +02:00
parent 10d09700fb
commit 26d755b895
No known key found for this signature in database
GPG Key ID: 6CFB948E15246897
3 changed files with 5 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/logical/storage_manager/match.rb
View File

@ -9,15 +9,15 @@ manager with no constraints as a default case.
StorageManager::Match.new do |matcher|
matcher.add_manager(type: :crop) do
StorageManager::SFTP.new("raikou3.donmai.us", base_url: "https://raikou3.donmai.us", hierarchical: true, base_dir: "/var/www/raikou3")
StorageManager::Local.new(hierarchical: true, base_dir: "/var/www/raikou3")
end
matcher.add_manager(id: 1..850_000) do
StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", hierarchical: true, base_dir: "/var/www/raikou1")
StorageManager::Local.new(hierarchical: true, base_dir: "/var/www/raikou1")
end
matcher.add_manager(id: 850_001..2_000_000) do
StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", hierarchical: true, base_dir: "/var/www/raikou2")
StorageManager::Local.new(hierarchical: true, base_dir: "/var/www/raikou2")
end
end

76
app/logical/storage_manager/sftp.rb
View File

@ -1,76 +0,0 @@
class StorageManager::Sftp < StorageManager
DEFAULT_PERMISSIONS = 0644
# http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start
DEFAULT_SSH_OPTIONS = {
timeout: 10,
logger: Rails.logger,
verbose: :fatal,
non_interactive: true,
}
attr_reader :hosts, :ssh_options
def initialize(*hosts, ssh_options: {}, **options)
@hosts = hosts
@ssh_options = DEFAULT_SSH_OPTIONS.merge(ssh_options)
super(**options)
end
def store(file, dest_path)
temp_upload_path = dest_path + "-" + SecureRandom.uuid + ".tmp"
dest_backup_path = dest_path + "-" + SecureRandom.uuid + ".bak"
each_host do |host, sftp|
begin
sftp.upload!(file.path, temp_upload_path)
sftp.setstat!(temp_upload_path, permissions: DEFAULT_PERMISSIONS)
# `rename!` can't overwrite existing files, so if a file already exists
# at dest_path we move it out of the way first.
force { sftp.rename!(dest_path, dest_backup_path) }
force { sftp.rename!(temp_upload_path, dest_path) }
rescue StandardError => e
# if anything fails, try to move the original file back in place (if it was moved).
force { sftp.rename!(dest_backup_path, dest_path) }
raise Error, e
ensure
force { sftp.remove!(temp_upload_path) }
force { sftp.remove!(dest_backup_path) }
end
end
end
def delete(dest_path)
each_host do |host, sftp|
force { sftp.remove!(dest_path) }
end
end
def open(dest_path)
file = Tempfile.new(binmode: true)
Net::SFTP.start(hosts.first, nil, ssh_options) do |sftp|
sftp.download!(dest_path, file.path)
end
file
end
protected
# Ignore "no such file" exceptions for the given operation.
def force
yield
rescue Net::SFTP::StatusException => e
raise Error, e unless e.description == "no such file"
end
def each_host
hosts.each do |host|
Net::SFTP.start(host, nil, ssh_options) do |sftp|
yield host, sftp
end
end
end
end

14
config/danbooru_default_config.rb
View File

@ -434,18 +434,12 @@ fart'
# hierarchical: true - store files in a hierarchical directory structure, based on the MD5 hash
StorageManager::Local.new(base_url: "#{CurrentUser.root_url}/", base_dir: "#{Rails.root}/public/data", hierarchical: true)
# Store files on one or more remote host(s). Configure SSH settings in
# ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("i1.example.com", "i2.example.com", base_dir: "/mnt/backup", hierarchical: false, ssh_options: {})
# Select the storage method based on the post's id and type (preview, large, or original).
# StorageManager::Hybrid.new do |id, md5, file_ext, type|
# ssh_options = { user: "danbooru" }
#
# if type.in?([:large, :original]) && id.in?(0..850_000)
# StorageManager::SFTP.new("raikou1.donmai.us", base_url: "https://raikou1.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
# StorageManager::Local.new(base_dir: "/path/to/files", hierarchical: true)
# else
# StorageManager::SFTP.new("raikou2.donmai.us", base_url: "https://raikou2.donmai.us", base_dir: "/path/to/files", hierarchical: true, ssh_options: ssh_options)
# StorageManager::Local.new(base_dir: "/path/to/files", hierarchical: true)
# end
# end
end
@ -457,10 +451,6 @@ fart'
# Backup files to /mnt/backup on the local filesystem.
# StorageManager::Local.new(base_dir: "/mnt/backup", hierarchical: false)
# Backup files to /mnt/backup on a remote system. Configure SSH settings
# in ~/.ssh_config or in the ssh_options param (ref: http://net-ssh.github.io/net-ssh/Net/SSH.html#method-c-start)
# StorageManager::SFTP.new("www.example.com", base_dir: "/mnt/backup", ssh_options: {})
end
#TAG CONFIGURATION