Libosmium  2.16.0
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2021 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/io/detail/read_write.hpp>
37 #include <osmium/io/error.hpp>
40 #include <osmium/util/file.hpp>
41 
42 #include <atomic>
43 #include <cerrno>
44 #include <cstddef>
45 #include <functional>
46 #include <map>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <tuple>
51 #include <utility>
52 
53 namespace osmium {
54 
55  namespace io {
56 
57  class Compressor {
58 
60 
61  protected:
62 
63  bool do_fsync() const noexcept {
64  return m_fsync == fsync::yes;
65  }
66 
67  public:
68 
69  explicit Compressor(const fsync sync) noexcept :
70  m_fsync(sync) {
71  }
72 
73  Compressor(const Compressor&) = default;
74  Compressor& operator=(const Compressor&) = default;
75 
76  Compressor(Compressor&&) noexcept = default;
77  Compressor& operator=(Compressor&&) noexcept = default;
78 
79  virtual ~Compressor() noexcept = default;
80 
81  virtual void write(const std::string& data) = 0;
82 
83  virtual void close() = 0;
84 
85  virtual std::size_t file_size() const {
86  return 0;
87  }
88 
89  }; // class Compressor
90 
91  class Decompressor {
92 
93  std::atomic<std::size_t> m_file_size{0};
94  std::atomic<std::size_t> m_offset{0};
95 
96  public:
97 
98  enum {
99  input_buffer_size = 1024U * 1024U
100  };
101 
102  Decompressor() = default;
103 
104  Decompressor(const Decompressor&) = delete;
106 
109 
110  virtual ~Decompressor() noexcept = default;
111 
112  virtual std::string read() = 0;
113 
114  virtual void close() = 0;
115 
116  std::size_t file_size() const noexcept {
117  return m_file_size;
118  }
119 
120  void set_file_size(const std::size_t size) noexcept {
121  m_file_size = size;
122  }
123 
124  std::size_t offset() const noexcept {
125  return m_offset;
126  }
127 
128  void set_offset(const std::size_t offset) noexcept {
129  m_offset = offset;
130  }
131 
132  }; // class Decompressor
133 
142 
143  public:
144 
145  using create_compressor_type = std::function<osmium::io::Compressor*(int, fsync)>;
147  using create_decompressor_type_buffer = std::function<osmium::io::Decompressor*(const char*, std::size_t)>;
148 
149  private:
150 
154 
155  using compression_map_type = std::map<const osmium::io::file_compression, callbacks_type>;
156 
158 
159  CompressionFactory() = default;
160 
162  const auto it = m_callbacks.find(compression);
163 
164  if (it != m_callbacks.end()) {
165  return it->second;
166  }
167 
168  std::string error_message{"Support for compression '"};
169  error_message += as_string(compression);
170  error_message += "' not compiled into this binary";
171  throw unsupported_file_format_error{error_message};
172  }
173 
174  public:
175 
178 
181 
182  ~CompressionFactory() noexcept = default;
183 
185  static CompressionFactory factory;
186  return factory;
187  }
188 
190  osmium::io::file_compression compression,
192  const create_decompressor_type_fd& create_decompressor_fd,
193  const create_decompressor_type_buffer& create_decompressor_buffer) {
194 
195  compression_map_type::value_type cc{compression,
196  std::make_tuple(create_compressor,
197  create_decompressor_fd,
198  create_decompressor_buffer)};
199 
200  return m_callbacks.insert(cc).second;
201  }
202 
203  template <typename... TArgs>
204  std::unique_ptr<osmium::io::Compressor> create_compressor(const osmium::io::file_compression compression, TArgs&&... args) const {
205  const auto callbacks = find_callbacks(compression);
206  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(callbacks)(std::forward<TArgs>(args)...));
207  }
208 
209  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const int fd) const {
210  const auto callbacks = find_callbacks(compression);
211  auto p = std::unique_ptr<osmium::io::Decompressor>(std::get<1>(callbacks)(fd));
212  p->set_file_size(osmium::file_size(fd));
213  return p;
214  }
215 
216  std::unique_ptr<osmium::io::Decompressor> create_decompressor(const osmium::io::file_compression compression, const char* buffer, const std::size_t size) const {
217  const auto callbacks = find_callbacks(compression);
218  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(callbacks)(buffer, size));
219  }
220 
221  }; // class CompressionFactory
222 
223  class NoCompressor final : public Compressor {
224 
225  std::size_t m_file_size = 0;
226  int m_fd;
227 
228  public:
229 
230  NoCompressor(const int fd, const fsync sync) :
231  Compressor(sync),
232  m_fd(fd) {
233  }
234 
235  NoCompressor(const NoCompressor&) = delete;
237 
240 
241  ~NoCompressor() noexcept override {
242  try {
243  close();
244  } catch (...) {
245  // Ignore any exceptions because destructor must not throw.
246  }
247  }
248 
249  void write(const std::string& data) override {
250  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
251  m_file_size += data.size();
252  }
253 
254  void close() override {
255  if (m_fd >= 0) {
256  const int fd = m_fd;
257  m_fd = -1;
258 
259  // Do not sync or close stdout
260  if (fd == 1) {
261  return;
262  }
263 
264  if (do_fsync()) {
265  osmium::io::detail::reliable_fsync(fd);
266  }
267  osmium::io::detail::reliable_close(fd);
268  }
269  }
270 
271  std::size_t file_size() const override {
272  return m_file_size;
273  }
274 
275  }; // class NoCompressor
276 
277  class NoDecompressor final : public Decompressor {
278 
279  int m_fd = -1;
280  const char* m_buffer = nullptr;
281  std::size_t m_buffer_size = 0;
282  std::size_t m_offset = 0;
283 
284  public:
285 
286  explicit NoDecompressor(const int fd) :
287  m_fd(fd) {
288  }
289 
290  NoDecompressor(const char* buffer, const std::size_t size) :
291  m_buffer(buffer),
292  m_buffer_size(size) {
293  }
294 
295  NoDecompressor(const NoDecompressor&) = delete;
297 
300 
301  ~NoDecompressor() noexcept override {
302  try {
303  close();
304  } catch (...) {
305  // Ignore any exceptions because destructor must not throw.
306  }
307  }
308 
309  std::string read() override {
310  std::string buffer;
311 
312  if (m_buffer) {
313  if (m_buffer_size != 0) {
314  const std::size_t size = m_buffer_size;
315  m_buffer_size = 0;
316  buffer.append(m_buffer, size);
317  }
318  } else {
320  const auto nread = detail::reliable_read(m_fd, &*buffer.begin(), osmium::io::Decompressor::input_buffer_size);
321  buffer.resize(std::string::size_type(nread));
322  }
323 
324  m_offset += buffer.size();
326 
327  return buffer;
328  }
329 
330  void close() override {
331  if (m_fd >= 0) {
332  const int fd = m_fd;
333  m_fd = -1;
334  osmium::io::detail::reliable_close(fd);
335  }
336  }
337 
338  }; // class NoDecompressor
339 
340  namespace detail {
341 
342  // we want the register_compression() function to run, setting
343  // the variable is only a side-effect, it will never be used
345  [](const int fd, const fsync sync) { return new osmium::io::NoCompressor{fd, sync}; },
346  [](const int fd) { return new osmium::io::NoDecompressor{fd}; },
347  [](const char* buffer, std::size_t size) { return new osmium::io::NoDecompressor{buffer, size}; }
348  );
349 
350  // dummy function to silence the unused variable warning from above
351  inline bool get_registered_no_compression() noexcept {
352  return registered_no_compression;
353  }
354 
355  } // namespace detail
356 
357  } // namespace io
358 
359 } // namespace osmium
360 
361 #endif // OSMIUM_IO_COMPRESSION_HPP
osmium::io::CompressionFactory::CompressionFactory
CompressionFactory(const CompressionFactory &)=delete
osmium::io::NoCompressor::~NoCompressor
~NoCompressor() noexcept override
Definition: compression.hpp:241
osmium::io::Decompressor::offset
std::size_t offset() const noexcept
Definition: compression.hpp:124
osmium::io::Compressor::operator=
Compressor & operator=(const Compressor &)=default
writer_options.hpp
osmium::io::Decompressor::Decompressor
Decompressor(Decompressor &&)=delete
osmium::io::NoCompressor::operator=
NoCompressor & operator=(const NoCompressor &)=delete
osmium::io::CompressionFactory::instance
static CompressionFactory & instance()
Definition: compression.hpp:184
osmium::io::Compressor::m_fsync
fsync m_fsync
Definition: compression.hpp:59
osmium::io::CompressionFactory::CompressionFactory
CompressionFactory()=default
osmium::io::Compressor::close
virtual void close()=0
osmium::io::fsync::yes
@ yes
osmium::io::Compressor::Compressor
Compressor(Compressor &&) noexcept=default
osmium::io::NoCompressor::file_size
std::size_t file_size() const override
Definition: compression.hpp:271
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(NoDecompressor &&)=delete
osmium::io::CompressionFactory::create_decompressor_type_buffer
std::function< osmium::io::Decompressor *(const char *, std::size_t)> create_decompressor_type_buffer
Definition: compression.hpp:147
detail
Definition: attr.hpp:342
osmium::io::NoDecompressor::~NoDecompressor
~NoDecompressor() noexcept override
Definition: compression.hpp:301
osmium::io::Compressor
Definition: compression.hpp:57
osmium::io::NoDecompressor::m_fd
int m_fd
Definition: compression.hpp:279
osmium::io::NoDecompressor::close
void close() override
Definition: compression.hpp:330
osmium::io::Decompressor
Definition: compression.hpp:91
osmium::io::NoDecompressor::operator=
NoDecompressor & operator=(const NoDecompressor &)=delete
osmium::io::NoCompressor::m_fd
int m_fd
Definition: compression.hpp:226
osmium::io::CompressionFactory::compression_map_type
std::map< const osmium::io::file_compression, callbacks_type > compression_map_type
Definition: compression.hpp:155
osmium::io::NoDecompressor::m_offset
std::size_t m_offset
Definition: compression.hpp:282
osmium::io::CompressionFactory::create_decompressor
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const char *buffer, const std::size_t size) const
Definition: compression.hpp:216
osmium::unsupported_file_format_error
Definition: error.hpp:56
osmium::io::Compressor::Compressor
Compressor(const fsync sync) noexcept
Definition: compression.hpp:69
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(const NoDecompressor &)=delete
osmium::io::Decompressor::operator=
Decompressor & operator=(Decompressor &&)=delete
osmium::io::NoCompressor::NoCompressor
NoCompressor(NoCompressor &&)=delete
osmium::io::Compressor::write
virtual void write(const std::string &data)=0
osmium::io::NoDecompressor
Definition: compression.hpp:277
osmium::io::CompressionFactory::m_callbacks
compression_map_type m_callbacks
Definition: compression.hpp:157
osmium::io::NoCompressor::m_file_size
std::size_t m_file_size
Definition: compression.hpp:225
osmium::io::Compressor::file_size
virtual std::size_t file_size() const
Definition: compression.hpp:85
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
osmium::io::Decompressor::input_buffer_size
@ input_buffer_size
Definition: compression.hpp:99
osmium::io::file_compression::none
@ none
osmium::io::NoCompressor
Definition: compression.hpp:223
osmium::io::NoCompressor::NoCompressor
NoCompressor(const int fd, const fsync sync)
Definition: compression.hpp:230
osmium::io::CompressionFactory::create_compressor
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition: compression.hpp:204
osmium::io::CompressionFactory::CompressionFactory
CompressionFactory(CompressionFactory &&)=delete
osmium::io::Decompressor::close
virtual void close()=0
osmium::io::CompressionFactory::create_decompressor
std::unique_ptr< osmium::io::Decompressor > create_decompressor(const osmium::io::file_compression compression, const int fd) const
Definition: compression.hpp:209
osmium::util::file_size
std::size_t file_size(int fd)
Definition: file.hpp:109
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(const char *buffer, const std::size_t size)
Definition: compression.hpp:290
osmium::io::Decompressor::read
virtual std::string read()=0
osmium::io::Decompressor::operator=
Decompressor & operator=(const Decompressor &)=delete
osmium::io::CompressionFactory::operator=
CompressionFactory & operator=(CompressionFactory &&)=delete
osmium::io::NoDecompressor::operator=
NoDecompressor & operator=(NoDecompressor &&)=delete
osmium::io::fsync
fsync
Definition: writer_options.hpp:51
osmium::io::CompressionFactory::callbacks_type
std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > callbacks_type
Definition: compression.hpp:153
osmium::io::file_compression
file_compression
Definition: file_compression.hpp:42
osmium::io::Decompressor::~Decompressor
virtual ~Decompressor() noexcept=default
osmium::io::NoCompressor::write
void write(const std::string &data) override
Definition: compression.hpp:249
osmium::io::as_string
const char * as_string(file_compression compression)
Definition: file_compression.hpp:48
osmium::io::NoCompressor::NoCompressor
NoCompressor(const NoCompressor &)=delete
osmium::io::Compressor::Compressor
Compressor(const Compressor &)=default
osmium::io::NoDecompressor::m_buffer_size
std::size_t m_buffer_size
Definition: compression.hpp:281
osmium::io::CompressionFactory::create_decompressor_type_fd
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:146
osmium::io::NoDecompressor::m_buffer
const char * m_buffer
Definition: compression.hpp:280
osmium::io::CompressionFactory::find_callbacks
const callbacks_type & find_callbacks(const osmium::io::file_compression compression) const
Definition: compression.hpp:161
std
Definition: location.hpp:551
osmium::io::NoDecompressor::read
std::string read() override
Definition: compression.hpp:309
osmium::io::CompressionFactory
Definition: compression.hpp:141
osmium::io::CompressionFactory::operator=
CompressionFactory & operator=(const CompressionFactory &)=delete
osmium::io::CompressionFactory::create_compressor_type
std::function< osmium::io::Compressor *(int, fsync)> create_compressor_type
Definition: compression.hpp:145
file.hpp
osmium::io::Decompressor::set_file_size
void set_file_size(const std::size_t size) noexcept
Definition: compression.hpp:120
file_compression.hpp
osmium::io::Decompressor::set_offset
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:128
osmium::io::CompressionFactory::~CompressionFactory
~CompressionFactory() noexcept=default
osmium::io::NoCompressor::operator=
NoCompressor & operator=(NoCompressor &&)=delete
osmium::io::CompressionFactory::register_compression
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:189
osmium::io::Decompressor::Decompressor
Decompressor()=default
osmium::io::Decompressor::m_file_size
std::atomic< std::size_t > m_file_size
Definition: compression.hpp:93
osmium::io::Compressor::do_fsync
bool do_fsync() const noexcept
Definition: compression.hpp:63
osmium::io::Decompressor::Decompressor
Decompressor(const Decompressor &)=delete
osmium::io::NoCompressor::close
void close() override
Definition: compression.hpp:254
error.hpp
osmium::io::Decompressor::file_size
std::size_t file_size() const noexcept
Definition: compression.hpp:116
osmium::io::NoDecompressor::NoDecompressor
NoDecompressor(const int fd)
Definition: compression.hpp:286
osmium::io::Decompressor::m_offset
std::atomic< std::size_t > m_offset
Definition: compression.hpp:94