Class: KamalBackup::Restic

Inherits:
Object
  • Object
show all
Defined in:
lib/kamal_backup/restic.rb

Constant Summary collapse

RESTIC_ENV_PATTERN = 
/\A(?:RESTIC_|AWS_|B2_|AZURE_|GOOGLE_|RCLONE_|OS_|ST_|HP_|HTTP_|HTTPS_|NO_PROXY)/i

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config, redactor:) ⇒ Restic

Returns a new instance of Restic.



13
14
15
16
 
# File 'lib/kamal_backup/restic.rb', line 13

def initialize(config, redactor:)
  @config = config
  @redactor = redactor
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



11
12
13
 
# File 'lib/kamal_backup/restic.rb', line 11

def config
  @config
end

#redactor ⇒ Object (readonly)

Returns the value of attribute redactor.



11
12
13
 
# File 'lib/kamal_backup/restic.rb', line 11

def redactor
  @redactor
end

Instance Method Details

#backup_file(path, filename:, tags:) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/kamal_backup/restic.rb', line 38

def backup_file(path, filename:, tags:)
  command = CommandSpec.new(
    argv: ["restic", "backup"] + host_args + ["--stdin", "--stdin-filename", filename] + tag_args(common_tags + tags),
    env: restic_env
  )
  log("backing up file content as #{filename}")

  File.open(path, "rb") do |file|
    output = Command.output
    context = output&.command_start(command, redactor: redactor)
    Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
      stdout_reader = Thread.new { Command.collect_stream(stdout, command_output: output, context: context, stream: :stdout, redactor: redactor) }
      stderr_reader = Thread.new { Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor) }
      IO.copy_stream(file, stdin)
      stdin.close
      out = stdout_reader.value
      err = stderr_reader.value
      status = wait_thread.value
      output&.command_exit(context, status.exitstatus)
      raise_command_error(command, status, out, err) unless status.success?

      CommandResult.new(stdout: out, stderr: err, status: status.exitstatus)
    end
  end
rescue Errno::ENOENT => e
  raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127, stderr: e.message)
end

#backup_path(path, tags:) ⇒ Object 



76
77
78
 
# File 'lib/kamal_backup/restic.rb', line 76

def backup_path(path, tags:)
  backup_paths([path], tags: tags)
end

#backup_paths(paths, tags:) ⇒ Object 



66
67
68
69
70
71
72
73
74
 
# File 'lib/kamal_backup/restic.rb', line 66

def backup_paths(paths, tags:)
  paths = Array(paths).compact.map(&:to_s).reject(&:empty?)

  if paths.any?
    path_tags = paths.map { |path| "path:#{config.backup_path_label(path)}" }
    log("backing up #{paths.size} file path(s): #{paths.join(", ")}")
    run(["backup"] + host_args + paths + tag_args(common_tags + tags + path_tags))
  end
end

#backup_stream(command, filename:, tags:) ⇒ Object 



29
30
31
32
33
34
35
36
 
# File 'lib/kamal_backup/restic.rb', line 29

def backup_stream(command, filename:, tags:)
  restic_command = CommandSpec.new(
    argv: ["restic", "backup"] + host_args + ["--stdin", "--stdin-filename", filename] + tag_args(common_tags + tags),
    env: restic_env
  )
  log("backing up stream as #{filename}")
  pipe_commands(command, restic_command, producer_label: "dump", consumer_label: "restic backup")
end

#check ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/kamal_backup/restic.rb', line 88

def check
  args = %w[check]
  args.concat(["--read-data-subset", config.check_read_data_subset]) if config.check_read_data_subset
  started_at = Time.now.utc
  result = run(args)
  write_last_check(status: "ok", started_at: started_at, finished_at: Time.now.utc, output: result.stdout)
  result
rescue CommandError => e
  write_last_check(status: "failed", started_at: started_at || Time.now.utc, finished_at: Time.now.utc, error: e.message)
  raise
end

#common_tags ⇒ Object 



193
194
195
 
# File 'lib/kamal_backup/restic.rb', line 193

def common_tags
  ["kamal-backup", "app:#{config.app_name}"]
end

#database_file(snapshot, adapter, database_name: nil) ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/kamal_backup/restic.rb', line 130

def database_file(snapshot, adapter, database_name: nil)
  legacy_prefix = "databases/#{config.app_name}/#{adapter}/"
  app = config.app_name.gsub(/[^A-Za-z0-9_.-]+/, "-")
  database = database_name.to_s.gsub(/[^A-Za-z0-9_.-]+/, "-")
  stable_prefix = database.empty? ? nil : "databases/#{app}/#{database}/#{adapter}."
  flat_prefix = "databases-#{app}-#{adapter}-"
  named_flat_prefix = database.empty? ? nil : "databases-#{app}-#{database}-#{adapter}-"
  ls_json(snapshot).find do |entry|
    next false unless entry["type"] == "file"

    normalized = entry["path"].to_s.sub(%r{\A/+}, "")
    (stable_prefix && normalized.start_with?(stable_prefix)) ||
      normalized.start_with?(legacy_prefix) ||
      File.basename(normalized).start_with?(flat_prefix) ||
      (named_flat_prefix && File.basename(normalized).start_with?(named_flat_prefix))
  end&.fetch("path")
end

#ensure_repository ⇒ Object 



18
19
20
21
22
23
24
25
26
27
 
# File 'lib/kamal_backup/restic.rb', line 18

def ensure_repository
  run(%w[snapshots --json], log_output: false)
rescue CommandError => e
  if config.restic_init_if_missing?
    log("restic repository not ready, running restic init")
    run(%w[init])
  else
    raise e
  end
end

#forget_after_success ⇒ Object 



80
81
82
83
84
85
86
 
# File 'lib/kamal_backup/restic.rb', line 80

def forget_after_success
  retention_tag_sets.each do |tags|
    args = ["forget", "--prune", "--group-by", "host"] + config.retention_args + filter_tag_args(tags)
    log("running restic forget/prune with retention policy for #{retention_scope(tags)}")
    run(args)
  end
end

#latest_snapshot(tags:) ⇒ Object 



114
115
116
117
118
119
 
# File 'lib/kamal_backup/restic.rb', line 114

def latest_snapshot(tags:)
  snapshots = snapshots_json(tags: common_tags + tags)
  snapshots.max_by { |snapshot| Time.parse(snapshot.fetch("time")) }
rescue JSON::ParserError
  nil
end

#ls_json(snapshot) ⇒ Object 



121
122
123
124
125
126
127
128
 
# File 'lib/kamal_backup/restic.rb', line 121

def ls_json(snapshot)
  output = run(["ls", "--json", snapshot], log_output: false).stdout
  output.lines.filter_map do |line|
    JSON.parse(line)
  rescue JSON::ParserError
    nil
  end
end

#pipe_dump_to_command(snapshot, filename, command) ⇒ Object 



148
149
150
151
 
# File 'lib/kamal_backup/restic.rb', line 148

def pipe_dump_to_command(snapshot, filename, command)
  restic_command = CommandSpec.new(argv: ["restic", "dump", snapshot, filename], env: restic_env)
  pipe_commands(restic_command, command, producer_label: "restic dump", consumer_label: command.argv.first)
end

#restore_snapshot(snapshot, target) ⇒ Object 



180
181
182
183
 
# File 'lib/kamal_backup/restic.rb', line 180

def restore_snapshot(snapshot, target)
  log("restoring file snapshot #{snapshot} to #{target}")
  run(["restore", snapshot, "--target", target])
end

#run(args, log_output: true) ⇒ Object 



185
186
187
188
189
190
191
 
# File 'lib/kamal_backup/restic.rb', line 185

def run(args, log_output: true)
  Command.capture(
    CommandSpec.new(argv: ["restic"] + args, env: restic_env),
    redactor: redactor,
    log_output: log_output
  )
end

#snapshots(tags: common_tags) ⇒ Object 



100
101
102
 
# File 'lib/kamal_backup/restic.rb', line 100

def snapshots(tags: common_tags)
  run(["snapshots"] + filter_tag_args(tags))
end

#snapshots_json(tags: common_tags) ⇒ Object 



104
105
106
107
108
109
110
111
112
 
# File 'lib/kamal_backup/restic.rb', line 104

def snapshots_json(tags: common_tags)
  output = run(["snapshots", "--json"] + filter_tag_args(tags), log_output: false).stdout
  snapshots = JSON.parse(output)
  required_tags = tags.compact
  snapshots.select do |snapshot|
    snapshot_tags = Array(snapshot["tags"])
    required_tags.all? { |tag| snapshot_tags.include?(tag) }
  end
end

#write_dump_to_path(snapshot, filename, target_path) ⇒ Object 



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/kamal_backup/restic.rb', line 153

def write_dump_to_path(snapshot, filename, target_path)
  command = CommandSpec.new(argv: ["restic", "dump", snapshot, filename], env: restic_env)
  target_path = File.expand_path(target_path)
  FileUtils.mkdir_p(File.dirname(target_path))
  temp_path = "#{target_path}.kamal-backup-#{$$}.tmp"

  output = Command.output
  context = output&.command_start(command, redactor: redactor)
  Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
    stdin.close
    stderr_reader = Thread.new { Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor) }
    File.open(temp_path, "wb") { |file| IO.copy_stream(stdout, file) }
    err = stderr_reader.value
    status = wait_thread.value
    output&.command_exit(context, status.exitstatus)
    raise_command_error(command, status, "", err) unless status.success?
  end
  File.rename(temp_path, target_path)
  target_path
rescue Errno::ENOENT => e
  FileUtils.rm_f(temp_path) if temp_path
  raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127, stderr: e.message)
rescue StandardError
  FileUtils.rm_f(temp_path) if temp_path
  raise
end