OGR
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_port.h"
36 #include "cpl_string.h"
37 #include "cpl_vsil_curl_priv.h"
38 #include "cpl_mem_cache.h"
39 
40 #include <curl/curl.h>
41 
42 #include <set>
43 #include <map>
44 #include <memory>
45 
47 
48 // 7.18.1
49 #if LIBCURL_VERSION_NUM >= 0x071201
50 #define HAVE_CURLINFO_REDIRECT_URL
51 #endif
52 
53 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
54 
55 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
56  const char * const* papszOptions);
57 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
58  struct curl_slist* poSrcToDestroy );
59 
60 namespace cpl {
61 
62 typedef enum
63 {
64  EXIST_UNKNOWN = -1,
65  EXIST_NO,
66  EXIST_YES,
67 } ExistStatus;
68 
69 class FileProp
70 {
71  public:
72  ExistStatus eExists = EXIST_UNKNOWN;
73  vsi_l_offset fileSize = 0;
74  time_t mTime = 0;
75  time_t nExpireTimestampLocal = 0;
76  CPLString osRedirectURL{};
77  bool bHasComputedFileSize = false;
78  bool bIsDirectory = false;
79  bool bS3LikeRedirect = false;
80  CPLString ETag{};
81 };
82 
83 typedef struct
84 {
85  bool bGotFileList = false;
86  CPLStringList oFileList{}; /* only file name without path */
87 } CachedDirList;
88 
89 typedef struct
90 {
91  char* pBuffer = nullptr;
92  size_t nSize = 0;
93  bool bIsHTTP = false;
94  bool bIsInHeader = false;
95  bool bMultiRange = false;
96  vsi_l_offset nStartOffset = 0;
97  vsi_l_offset nEndOffset = 0;
98  int nHTTPCode = 0;
99  vsi_l_offset nContentLength = 0;
100  bool bFoundContentRange = false;
101  bool bError = false;
102  bool bDownloadHeaderOnly = false;
103  bool bDetectRangeDownloadingError = false;
104  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
105 
106  VSILFILE *fp = nullptr;
107  VSICurlReadCbkFunc pfnReadCbk = nullptr;
108  void *pReadCbkUserData = nullptr;
109  bool bInterrupted = false;
110 
111 #if LIBCURL_VERSION_NUM < 0x073600
112  // Workaround to ignore extra HTTP response headers from
113  // proxies in older versions of curl.
114  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
115  bool bIsProxyConnectHeader = false;
116 #endif
117 } WriteFuncStruct;
118 
119 /************************************************************************/
120 /* VSICurlFilesystemHandler */
121 /************************************************************************/
122 
123 class VSICurlHandle;
124 
125 class VSICurlFilesystemHandler : public VSIFilesystemHandler
126 {
127  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
128 
129  struct FilenameOffsetPair
130  {
131  std::string filename_;
132  vsi_l_offset offset_;
133 
134  FilenameOffsetPair(const std::string& filename,
135  vsi_l_offset offset) :
136  filename_(filename), offset_(offset) {}
137 
138  bool operator==(const FilenameOffsetPair& other) const
139  {
140  return filename_ == other.filename_ &&
141  offset_ == other.offset_;
142  }
143  };
144  struct FilenameOffsetPairHasher
145  {
146  std::size_t operator()(const FilenameOffsetPair& k) const
147  {
148  return std::hash<std::string>()(k.filename_) ^
149  std::hash<vsi_l_offset>()(k.offset_);
150  }
151  };
152 
153  using RegionCacheType =
154  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
155  lru11::NullLock,
156  std::unordered_map<
157  FilenameOffsetPair,
158  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
159  std::shared_ptr<std::string>>>::iterator,
160  FilenameOffsetPairHasher>>;
161 
162  RegionCacheType oRegionCache;
163 
164  lru11::Cache<std::string, FileProp> oCacheFileProp;
165 
166  int nCachedFilesInDirList = 0;
167  lru11::Cache<std::string, CachedDirList> oCacheDirList;
168 
169  char** ParseHTMLFileList(const char* pszFilename,
170  int nMaxFiles,
171  char* pszData,
172  bool* pbGotFileList);
173 
174 protected:
175  CPLMutex *hMutex = nullptr;
176 
177  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
178  virtual char** GetFileList(const char *pszFilename,
179  int nMaxFiles,
180  bool* pbGotFileList);
181 
182  void RegisterEmptyDir( const CPLString& osDirname );
183 
184  bool AnalyseS3FileList( const CPLString& osBaseURL,
185  const char* pszXML,
186  CPLStringList& osFileList,
187  int nMaxFiles,
188  bool bIgnoreGlacierStorageClass,
189  bool& bIsTruncated );
190 
191  void AnalyseSwiftFileList( const CPLString& osBaseURL,
192  const CPLString& osPrefix,
193  const char* pszJson,
194  CPLStringList& osFileList,
195  int nMaxFilesThisQuery,
196  int nMaxFiles,
197  bool& bIsTruncated,
198  CPLString& osNextMarker );
199 
200  static const char* GetOptionsStatic();
201 
202  static bool IsAllowedFilename( const char* pszFilename );
203 
204 public:
205  VSICurlFilesystemHandler();
206  ~VSICurlFilesystemHandler() override;
207 
208  VSIVirtualHandle *Open( const char *pszFilename,
209  const char *pszAccess,
210  bool bSetError ) override;
211 
212  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
213  int nFlags ) override;
214  int Unlink( const char *pszFilename ) override;
215  int Rename( const char *oldpath, const char *newpath ) override;
216  int Mkdir( const char *pszDirname, long nMode ) override;
217  int Rmdir( const char *pszDirname ) override;
218  char **ReadDir( const char *pszDirname ) override
219  { return ReadDirEx(pszDirname, 0); }
220  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
221 
222  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
223  override { return true; }
224 
225  const char* GetActualURL(const char* pszFilename) override;
226 
227  const char* GetOptions() override;
228 
229  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
230  bool* pbGotFileList );
231  void InvalidateDirContent( const char *pszDirname );
232 
233  virtual CPLString GetFSPrefix() { return "/vsicurl/"; }
234  virtual bool AllowCachedDataFor(const char* pszFilename);
235 
236  std::shared_ptr<std::string> GetRegion( const char* pszURL,
237  vsi_l_offset nFileOffsetStart );
238 
239  void AddRegion( const char* pszURL,
240  vsi_l_offset nFileOffsetStart,
241  size_t nSize,
242  const char *pData );
243 
244  bool GetCachedFileProp( const char* pszURL,
245  FileProp& oFileProp );
246  void SetCachedFileProp( const char* pszURL,
247  const FileProp& oFileProp );
248  void InvalidateCachedData( const char* pszURL );
249 
250  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
251 
252  virtual void ClearCache();
253  virtual void PartialClearCache(const char* pszFilename);
254 
255 
256  bool GetCachedDirList( const char* pszURL,
257  CachedDirList& oCachedDirList );
258  void SetCachedDirList( const char* pszURL,
259  const CachedDirList& oCachedDirList );
260  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
261 
262  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
263 };
264 
265 /************************************************************************/
266 /* VSICurlHandle */
267 /************************************************************************/
268 
269 class VSICurlHandle : public VSIVirtualHandle
270 {
271  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
272 
273  protected:
274  VSICurlFilesystemHandler* poFS = nullptr;
275 
276  bool m_bCached = true;
277 
278  FileProp oFileProp{};
279 
280  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
281  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
282 
283  char **m_papszHTTPOptions = nullptr;
284 
285  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
286  int nBlocksToDownload = 1;
287 
288  bool bStopOnInterruptUntilUninstall = false;
289  bool bInterrupted = false;
290  VSICurlReadCbkFunc pfnReadCbk = nullptr;
291  void *pReadCbkUserData = nullptr;
292 
293  int m_nMaxRetry = 0;
294  double m_dfRetryDelay = 0.0;
295 
296  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
297  const int nBlocks,
298  const char* pBuffer,
299  size_t nSize );
300 
301  private:
302 
303  vsi_l_offset curOffset = 0;
304 
305  bool bEOF = false;
306 
307  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
308 
309  bool m_bUseHead = false;
310 
311  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
312  const vsi_l_offset* panOffsets,
313  const size_t* panSizes );
314  CPLString GetRedirectURLIfValid(bool& bHasExpired);
315 
316  protected:
317  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
318  const struct curl_slist* /* psExistingHeaders */)
319  { return nullptr; }
320  virtual bool AllowAutomaticRedirection() { return true; }
321  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
322  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
323  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
324  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
325  void SetURL(const char* pszURL);
326 
327  public:
328 
329  VSICurlHandle( VSICurlFilesystemHandler* poFS,
330  const char* pszFilename,
331  const char* pszURLIn = nullptr );
332  ~VSICurlHandle() override;
333 
334  int Seek( vsi_l_offset nOffset, int nWhence ) override;
335  vsi_l_offset Tell() override;
336  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
337  int ReadMultiRange( int nRanges, void ** ppData,
338  const vsi_l_offset* panOffsets,
339  const size_t* panSizes ) override;
340  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
341  int Eof() override;
342  int Flush() override;
343  int Close() override;
344 
345  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
346  vsi_l_offset GetFileSize() { return GetFileSize(false); }
347  virtual vsi_l_offset GetFileSize( bool bSetError );
348  bool Exists( bool bSetError );
349  bool IsDirectory() const { return oFileProp.bIsDirectory; }
350  time_t GetMTime() const { return oFileProp.mTime; }
351 
352  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
353  void* pfnUserData,
354  int bStopOnInterruptUntilUninstall );
355  int UninstallReadCbk();
356 
357  const char *GetURL() const { return m_pszURL; }
358 };
359 
360 /************************************************************************/
361 /* IVSIS3LikeFSHandler */
362 /************************************************************************/
363 
364 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
365 {
366  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
367 
368  protected:
369  char** GetFileList( const char *pszFilename,
370  int nMaxFiles,
371  bool* pbGotFileList ) override;
372 
373  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
374  const char* pszURI, bool bAllowNoObject) = 0;
375 
376  IVSIS3LikeFSHandler() = default;
377 
378  public:
379  int Unlink( const char *pszFilename ) override;
380  int Mkdir( const char *pszDirname, long nMode ) override;
381  int Rmdir( const char *pszDirname ) override;
382  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
383  int nFlags ) override;
384 
385  virtual int DeleteObject( const char *pszFilename );
386 
387  virtual const char* GetDebugKey() const = 0;
388 
389  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
390  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
391 
392  bool Sync( const char* pszSource, const char* pszTarget,
393  const char* const * papszOptions,
394  GDALProgressFunc pProgressFunc,
395  void *pProgressData,
396  char*** ppapszOutputs ) override;
397 
398  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
399  const char* const *papszOptions) override;
400 };
401 
402 /************************************************************************/
403 /* IVSIS3LikeHandle */
404 /************************************************************************/
405 
406 class IVSIS3LikeHandle: public VSICurlHandle
407 {
408  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
409 
410  protected:
411  bool UseLimitRangeGetInsteadOfHead() override { return true; }
412  bool IsDirectoryFromExists( const char* pszVerb,
413  int response_code ) override
414  {
415  // A bit dirty, but on S3, a GET on a existing directory returns a 416
416  return response_code == 416 && EQUAL(pszVerb, "GET") &&
417  CPLString(m_pszURL).back() == '/';
418  }
419  void ProcessGetFileSizeResult( const char* pszContent ) override
420  {
421  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
422  }
423 
424  public:
425  IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
426  const char* pszFilename,
427  const char* pszURLIn = nullptr ) :
428  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
429  ~IVSIS3LikeHandle() override {}
430 };
431 
432 /************************************************************************/
433 /* VSIS3WriteHandle */
434 /************************************************************************/
435 
436 class VSIS3WriteHandle final : public VSIVirtualHandle
437 {
438  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
439 
440  IVSIS3LikeFSHandler *m_poFS = nullptr;
441  CPLString m_osFilename{};
442  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
443  bool m_bUseChunked = false;
444 
445  vsi_l_offset m_nCurOffset = 0;
446  int m_nBufferOff = 0;
447  int m_nBufferSize = 0;
448  int m_nBufferOffReadCallback = 0;
449  bool m_bClosed = false;
450  GByte *m_pabyBuffer = nullptr;
451  CPLString m_osUploadID{};
452  int m_nPartNumber = 0;
453  std::vector<CPLString> m_aosEtags{};
454  CPLString m_osXML{};
455  int m_nOffsetInXML = 0;
456  bool m_bError = false;
457 
458  CURLM *m_hCurlMulti = nullptr;
459  CURL *m_hCurl = nullptr;
460  const void *m_pBuffer = nullptr;
461  CPLString m_osCurlErrBuf{};
462  size_t m_nChunkedBufferOff = 0;
463  size_t m_nChunkedBufferSize = 0;
464 
465  int m_nMaxRetry = 0;
466  double m_dfRetryDelay = 0.0;
467  WriteFuncStruct m_sWriteFuncHeaderData{};
468 
469  static size_t ReadCallBackBuffer( char *buffer, size_t size,
470  size_t nitems, void *instream );
471  bool InitiateMultipartUpload();
472  bool UploadPart();
473  static size_t ReadCallBackXML( char *buffer, size_t size,
474  size_t nitems, void *instream );
475  bool CompleteMultipart();
476  bool AbortMultipart();
477  bool DoSinglePartPUT();
478 
479  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
480  size_t nitems, void *instream );
481  size_t WriteChunked( const void *pBuffer,
482  size_t nSize, size_t nMemb );
483  int FinishChunkedTransfer();
484 
485  void InvalidateParentDirectory();
486 
487  public:
488  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
489  const char* pszFilename,
490  IVSIS3LikeHandleHelper* poS3HandleHelper,
491  bool bUseChunked );
492  ~VSIS3WriteHandle() override;
493 
494  int Seek( vsi_l_offset nOffset, int nWhence ) override;
495  vsi_l_offset Tell() override;
496  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
497  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
498  int Eof() override;
499  int Close() override;
500 
501  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
502 };
503 
504 /************************************************************************/
505 /* VSIAppendWriteHandle */
506 /************************************************************************/
507 
508 class VSIAppendWriteHandle : public VSIVirtualHandle
509 {
510  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
511 
512  protected:
513 
514  VSICurlFilesystemHandler* m_poFS = nullptr;
515  CPLString m_osFSPrefix{};
516  CPLString m_osFilename{};
517 
518  vsi_l_offset m_nCurOffset = 0;
519  int m_nBufferOff = 0;
520  int m_nBufferSize = 0;
521  int m_nBufferOffReadCallback = 0;
522  bool m_bClosed = false;
523  GByte *m_pabyBuffer = nullptr;
524  bool m_bError = false;
525 
526  static size_t ReadCallBackBuffer( char *buffer, size_t size,
527  size_t nitems, void *instream );
528  virtual bool Send(bool bIsLastBlock) = 0;
529 
530  public:
531  VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
532  const char* pszFSPrefix,
533  const char* pszFilename,
534  int nChunkSize );
535  virtual ~VSIAppendWriteHandle();
536 
537  int Seek( vsi_l_offset nOffset, int nWhence ) override;
538  vsi_l_offset Tell() override;
539  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
540  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
541  int Eof() override;
542  int Close() override;
543 
544  bool IsOK() { return m_pabyBuffer != nullptr; }
545 };
546 
547 int VSICURLGetDownloadChunkSize();
548 
549 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
550  VSILFILE *fp,
551  VSICurlReadCbkFunc pfnReadCbk,
552  void *pReadCbkUserData );
553 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
554  size_t nmemb, void *req );
555 void MultiPerform(CURLM* hCurlMultiHandle,
556  CURL* hEasyHandle = nullptr);
557 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
558 
559 } // namespace cpl
560 
562 
563 #endif // HAVE_CURL
564 
565 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
VSIFPutcL
int VSIFPutcL(int, VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Write a single byte to the file.
Definition: cpl_vsil.cpp:1955
VSIGetNextDirEntry
const VSIDIREntry * VSIGetNextDirEntry(VSIDIR *dir)
Return the next entry of the directory.
Definition: cpl_vsil.cpp:379
VSIOpenDir
VSIDIR * VSIOpenDir(const char *pszPath, int nRecurseDepth, const char *const *papszOptions)
Open a directory to read its entries.
Definition: cpl_vsil.cpp:342
VSIOverwriteFile
int VSIOverwriteFile(VSILFILE *fpTarget, const char *pszSourceFilename)
Overwrite an existing file with content from another one.
Definition: cpl_vsil.cpp:2225
VSIInstallSwiftFileHandler
void VSIInstallSwiftFileHandler(void)
Install /vsiswift/ OpenStack Swif Object Storage (Swift) file system handler (requires libcurl)
Definition: cpl_vsil_swift.cpp:706
VSIRmdirRecursive
int VSIRmdirRecursive(const char *pszDirname)
Delete a directory recursively.
Definition: cpl_vsil.cpp:669
CXT_Element
@ CXT_Element
Definition: cpl_minixml.h:46
VSIInstallAzureFileHandler
void VSIInstallAzureFileHandler(void)
Install /vsiaz/ Microsoft Azure Blob file system handler (requires libcurl)
Definition: cpl_vsil_az.cpp:1219
CSLTokenizeString2
char ** CSLTokenizeString2(const char *pszString, const char *pszDelimiter, int nCSLTFlags)
Definition: cpl_string.cpp:836
CPLGetLastErrorMsg
const char * CPLGetLastErrorMsg(void)
Definition: cpl_error.cpp:866
GByte
unsigned char GByte
Definition: cpl_port.h:215
VSIFReadL
size_t VSIFReadL(void *, size_t, size_t, VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Read bytes from file.
Definition: cpl_vsil.cpp:1690
VSIRmdir
int VSIRmdir(const char *pszDirname)
Delete a directory.
Definition: cpl_vsil.cpp:647
cpl_error.h
CPL_LSBWORD16
#define CPL_LSBWORD16(x)
Definition: cpl_port.h:822
CPLEscapeString
char * CPLEscapeString(const char *pszString, int nLength, int nScheme)
Definition: cpl_string.cpp:2036
VSIInstallWebHdfsHandler
void VSIInstallWebHdfsHandler(void)
Install /vsiwebhdfs/ WebHDFS (Hadoop File System) REST API file system handler (requires libcurl)
Definition: cpl_vsil_webhdfs.cpp:1188
VSIVirtualHandle::Truncate
virtual int Truncate(vsi_l_offset nNewSize)
Truncate/expand the file to the specified size.
CPLString::Printf
CPLSTRING_METHOD_DLL CPLString & Printf(const char *pszFormat,...)
Definition: cplstring.cpp:59
VSIInstallStdoutHandler
void VSIInstallStdoutHandler(void)
Install /vsistdout/ file system handler.
Definition: cpl_vsil_stdout.cpp:444
VSIGetSignedURL
char * VSIGetSignedURL(const char *pszFilename, CSLConstList papszOptions)
Returns a signed URL of a supplied filename.
Definition: cpl_vsil.cpp:959
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:438
CPLCalloc
void * CPLCalloc(size_t, size_t)
Definition: cpl_conv.cpp:138
VSICloseDir
void VSICloseDir(VSIDIR *dir)
Close a directory.
Definition: cpl_vsil.cpp:398
VSIInstallAzureStreamingFileHandler
void VSIInstallAzureStreamingFileHandler(void)
Install /vsiaz_streaming/ Microsoft Azure Blob file system handler (requires libcurl)
Definition: cpl_vsil_curl_streaming.cpp:2139
VSIRealloc
void * VSIRealloc(void *, size_t)
Definition: cpl_vsisimple.cpp:681
cpl_minixml.h
CPLPopErrorHandler
void CPLPopErrorHandler(void)
Definition: cpl_error.cpp:1242
VSIInstallSubFileHandler
void VSIInstallSubFileHandler(void)
Definition: cpl_vsil_subfile.cpp:492
cpl_vsi.h
CSLFetchNameValueDef
const char * CSLFetchNameValueDef(CSLConstList papszStrList, const char *pszName, const char *pszDefault)
Definition: cpl_string.cpp:1646
CSLSetNameValue
char ** CSLSetNameValue(char **papszStrList, const char *pszName, const char *pszValue)
Definition: cpl_string.cpp:1877
VSIDIREntry::pszName
char * pszName
Definition: cpl_vsi.h:319
VSIDIREntry
struct VSIDIREntry VSIDIREntry
CPLAtoGIntBig
GIntBig CPLAtoGIntBig(const char *pszString)
Definition: cpl_conv.cpp:998
VSIRangeStatus
VSIRangeStatus
Definition: cpl_vsi.h:175
CSLCount
int CSLCount(CSLConstList papszStrList)
Definition: cpl_string.cpp:147
VSI_ISREG
#define VSI_ISREG(x)
Definition: cpl_vsi.h:124
VSIMkdir
int VSIMkdir(const char *pszPathname, long mode)
Create a directory.
Definition: cpl_vsil.cpp:423
VSIVirtualHandle::Close
virtual int Close()=0
Close file.
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:329
CPLStringList::size
int size() const
Definition: cpl_string.h:460
VSIRename
int VSIRename(const char *oldpath, const char *newpath)
Rename a file.
Definition: cpl_vsil.cpp:529
VSIMkdirRecursive
int VSIMkdirRecursive(const char *pszPathname, long mode)
Create a directory and all its ancestors.
Definition: cpl_vsil.cpp:446
VSIFOpenExL
VSILFILE * VSIFOpenExL(const char *, const char *, int)
Open file.
Definition: cpl_vsil.cpp:1416
VSISync
int VSISync(const char *pszSource, const char *pszTarget, const char *const *papszOptions, GDALProgressFunc pProgressFunc, void *pProgressData, char ***ppapszOutputs)
Synchronize a source file/directory with a target file/directory.
Definition: cpl_vsil.cpp:595
VSIRewindL
void VSIRewindL(VSILFILE *)
Rewind the file pointer to the beginning of the file.
Definition: cpl_vsil.cpp:1597
VSI_STAT_NATURE_FLAG
#define VSI_STAT_NATURE_FLAG
Definition: cpl_vsi.h:205
CPLParseXMLString
CPLXMLNode * CPLParseXMLString(const char *)
Parse an XML string into tree form.
Definition: cpl_minixml.cpp:614
CPL_LSBPTR64
#define CPL_LSBPTR64(x)
Definition: cpl_port.h:838
CPLFetchBool
bool CPLFetchBool(CSLConstList papszStrList, const char *pszKey, bool bDefault)
Definition: cpl_string.cpp:1600
VSIIsCaseSensitiveFS
int VSIIsCaseSensitiveFS(const char *pszFilename)
Returns if the filenames of the filesystem are case sensitive.
Definition: cpl_vsil.cpp:826
VSIFWriteL
size_t VSIFWriteL(const void *, size_t, size_t, VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Write bytes to file.
Definition: cpl_vsil.cpp:1804
VSIVirtualHandle::Flush
virtual int Flush()
Flush pending writes to disk.
Definition: cpl_vsi_virtual.h:97
CPLRealloc
void * CPLRealloc(void *, size_t)
Definition: cpl_conv.cpp:225
VSIVirtualHandle::ReadMultiRange
virtual int ReadMultiRange(int nRanges, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes)
Read several ranges of bytes from file.
VSIHasOptimizedReadMultiRange
int VSIHasOptimizedReadMultiRange(const char *pszPath)
Returns if the filesystem supports efficient multi-range reading.
Definition: cpl_vsil.cpp:882
VSI_ISDIR
#define VSI_ISDIR(x)
Definition: cpl_vsi.h:126
CPLXMLNode
Definition: cpl_minixml.h:66
CPL_FRMT_GIB
#define CPL_FRMT_GIB
Definition: cpl_port.h:316
EQUAL
#define EQUAL(a, b)
Definition: cpl_port.h:569
CPLGetPath
const char * CPLGetPath(const char *)
Definition: cpl_path.cpp:153
VSIFCloseL
int VSIFCloseL(VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Close file.
Definition: cpl_vsil.cpp:1472
VSIVirtualHandle::Tell
virtual vsi_l_offset Tell()=0
Tell current file offset.
cpl_http.h
VSIInstallSparseFileHandler
void VSIInstallSparseFileHandler(void)
Definition: cpl_vsil_sparsefile.cpp:547
CPLStringList::AddString
CPLStringList & AddString(const char *pszNewString)
Definition: cplstringlist.cpp:333
CPLPushErrorHandler
void CPLPushErrorHandler(CPLErrorHandler)
Definition: cpl_error.cpp:1185
CPLAssert
#define CPLAssert(expr)
Definition: cpl_error.h:186
VSIGetFileSystemOptions
const char * VSIGetFileSystemOptions(const char *pszFilename)
Return the list of options associated with a virtual file system handler as a serialized XML string.
Definition: cpl_vsil.cpp:2360
CPL_C_START
#define CPL_C_START
Definition: cpl_port.h:337
VSIFGetRangeStatusL
VSIRangeStatus VSIFGetRangeStatusL(VSILFILE *fp, vsi_l_offset nStart, vsi_l_offset nLength)
Return if a given file range contains data or holes filled with zeroes.
Definition: cpl_vsil.cpp:2007
VSIFEofL
int VSIFEofL(VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Test for end of file.
Definition: cpl_vsil.cpp:1850
VSIDIREntry::papszExtra
char ** papszExtra
Definition: cpl_vsi.h:333
CPLE_OutOfMemory
#define CPLE_OutOfMemory
Definition: cpl_error.h:101
VSIInstallS3FileHandler
void VSIInstallS3FileHandler(void)
Install /vsis3/ Amazon S3 file system handler (requires libcurl)
Definition: cpl_vsil_s3.cpp:2750
CPLTestBool
bool CPLTestBool(const char *pszValue)
Definition: cpl_string.cpp:1526
VSIInstallGZipFileHandler
void VSIInstallGZipFileHandler(void)
Install GZip file system handler.
Definition: cpl_vsil_gzip.cpp:2449
CPLGetXMLNode
CPLXMLNode * CPLGetXMLNode(CPLXMLNode *poRoot, const char *pszPath)
Find node by path.
Definition: cpl_minixml.cpp:1550
VSIReadDirEx
char ** VSIReadDirEx(const char *pszPath, int nMaxFiles)
Read names in a directory.
Definition: cpl_vsil.cpp:117
VSIVirtualHandle::Read
virtual size_t Read(void *pBuffer, size_t nSize, size_t nCount)=0
Read bytes from file.
CSLConstList
char ** CSLConstList
Definition: cpl_port.h:1194
CPL_FRMT_GUIB
#define CPL_FRMT_GUIB
Definition: cpl_port.h:318
VSIFTellL
vsi_l_offset VSIFTellL(VSILFILE *)
Tell current file offset.
Definition: cpl_vsil.cpp:1575
CPLES_URL
#define CPLES_URL
Definition: cpl_string.h:171
CPLGetDirname
const char * CPLGetDirname(const char *)
Definition: cpl_path.cpp:208
GUIntBig
unsigned long long GUIntBig
Definition: cpl_port.h:251
CPLReadDir
#define CPLReadDir
Definition: cpl_vsi.h:303
CPLBase64Encode
char * CPLBase64Encode(int nBytes, const GByte *pabyData)
Definition: cpl_base64.cpp:199
CPLStringList::empty
bool empty() const
Definition: cpl_string.h:464
VSIFFlushL
int VSIFFlushL(VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Flush pending writes to disk.
Definition: cpl_vsil.cpp:1638
CPLDestroyXMLNode
void CPLDestroyXMLNode(CPLXMLNode *)
Destroy a tree.
Definition: cpl_minixml.cpp:1405
VSIInstallGSFileHandler
void VSIInstallGSFileHandler(void)
Install /vsigs/ Google Cloud Storage file system handler (requires libcurl)
Definition: cpl_vsil_gs.cpp:329
CPL_C_END
#define CPL_C_END
Definition: cpl_port.h:339
CPLDebug
void CPLDebug(const char *, const char *,...)
Definition: cpl_error.cpp:602
VSIInstallStdinHandler
void VSIInstallStdinHandler(void)
Install /vsistdin/ file system handler.
Definition: cpl_vsil_stdin.cpp:421
VSIVirtualHandle::GetRangeStatus
virtual VSIRangeStatus GetRangeStatus(vsi_l_offset nOffset, vsi_l_offset nLength)
Return if a given file range contains data or holes filled with zeroes.
Definition: cpl_vsi_virtual.h:102
VSIInstallHdfsHandler
void VSIInstallHdfsHandler(void)
Install /vsihdfs/ file system handler (non-functional stub)
Definition: cpl_vsil_hdfs.cpp:431
CPLFormFilename
const char * CPLFormFilename(const char *pszPath, const char *pszBasename, const char *pszExtension)
Definition: cpl_path.cpp:537
VSIDIREntry
Definition: cpl_vsi.h:316
VSIStatL
int VSIStatL(const char *, VSIStatBufL *)
Get filesystem object info.
Definition: cpl_vsil.cpp:737
VSIUnlink
int VSIUnlink(const char *pszFilename)
Delete a file.
Definition: cpl_vsil.cpp:498
STARTS_WITH
#define STARTS_WITH(a, b)
Definition: cpl_port.h:578
CPLMalloc
void * CPLMalloc(size_t)
Definition: cpl_conv.cpp:168
VSIInstallCryptFileHandler
void VSIInstallCryptFileHandler(void)
Install /vsicrypt/ encrypted file system handler (requires libcrypto++)
Definition: cpl_vsil_crypt.cpp:2024
VSIInstallGSStreamingFileHandler
void VSIInstallGSStreamingFileHandler(void)
Install /vsigs_streaming/ Google Cloud Storage file system handler (requires libcurl)
Definition: cpl_vsil_curl_streaming.cpp:2121
VSISupportsSparseFiles
int VSISupportsSparseFiles(const char *pszPath)
Returns if the filesystem supports sparse files.
Definition: cpl_vsil.cpp:856
VSIFTruncateL
int VSIFTruncateL(VSILFILE *, vsi_l_offset) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Truncate/expand the file to the specified size.
Definition: cpl_vsil.cpp:1892
VSIGetFileSystemsPrefixes
char ** VSIGetFileSystemsPrefixes(void)
Return the list of prefixes for virtual file system handlers currently registered.
Definition: cpl_vsil.cpp:2340
CSLFindString
int CSLFindString(CSLConstList papszList, const char *pszTarget)
Definition: cpl_string.cpp:670
VSIInstallCurlStreamingFileHandler
void VSIInstallCurlStreamingFileHandler(void)
Install /vsicurl_streaming/ HTTP/FTP file system handler (requires libcurl).
Definition: cpl_vsil_curl_streaming.cpp:2084
cpl_conv.h
VSIInstallCurlFileHandler
void VSIInstallCurlFileHandler(void)
Install /vsicurl/ HTTP/FTP file system handler (requires libcurl)
Definition: cpl_vsil_curl.cpp:4300
cpl_string.h
VSIInstallTarFileHandler
void VSIInstallTarFileHandler(void)
Install /vsitar/ file system handler.
Definition: cpl_vsil_tar.cpp:619
CPLErrorReset
void CPLErrorReset(void)
Definition: cpl_error.cpp:729
VSIFSeekL
int VSIFSeekL(VSILFILE *, vsi_l_offset, int) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Seek to requested offset.
Definition: cpl_vsil.cpp:1532
CPLSPrintf
const char * CPLSPrintf(const char *fmt,...)
Definition: cpl_string.cpp:977
VSIVirtualHandle::Write
virtual size_t Write(const void *pBuffer, size_t nSize, size_t nCount)=0
Write bytes to file.
CPLGetXMLValue
const char * CPLGetXMLValue(const CPLXMLNode *poRoot, const char *pszPath, const char *pszDefault)
Fetch element/attribute value.
Definition: cpl_minixml.cpp:1638
CPLMutexHolder
Definition: cpl_multiproc.h:139
CPLError
void CPLError(CPLErr eErrClass, CPLErrorNum err_no, const char *fmt,...)
Definition: cpl_error.cpp:308
CPLParseNameValue
const char * CPLParseNameValue(const char *pszNameValue, char **ppszKey)
Definition: cpl_string.cpp:1754
VSIInstallMemFileHandler
void VSIInstallMemFileHandler(void)
Install "memory" file system handler.
Definition: cpl_vsi_mem.cpp:868
VSIInstallZipFileHandler
void VSIInstallZipFileHandler(void)
Install ZIP file system handler.
Definition: cpl_vsil_gzip.cpp:3320
vsi_l_offset
GUIntBig vsi_l_offset
Definition: cpl_vsi.h:140
VSIFree
void VSIFree(void *)
Definition: cpl_vsisimple.cpp:827
CPLE_NotSupported
#define CPLE_NotSupported
Definition: cpl_error.h:109
VSIInstallOSSFileHandler
void VSIInstallOSSFileHandler(void)
Install /vsioss/ Alibaba Cloud Object Storage Service (OSS) file system handler (requires libcurl)
Definition: cpl_vsil_oss.cpp:388
CPLUnescapeString
char * CPLUnescapeString(const char *pszString, int *pnLength, int nScheme)
Definition: cpl_string.cpp:2248
VSIFReadMultiRangeL
int VSIFReadMultiRangeL(int nRanges, void **ppData, const vsi_l_offset *panOffsets, const size_t *panSizes, VSILFILE *) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Read several ranges of bytes from file.
Definition: cpl_vsil.cpp:1750
CPLSetConfigOption
void CPLSetConfigOption(const char *, const char *)
Definition: cpl_conv.cpp:1832
CPL_ARRAYSIZE
#define CPL_ARRAYSIZE(array)
Definition: cpl_port.h:1045
end
OGRLayer::FeatureIterator end(OGRLayer *poLayer)
Definition: ogrsf_frmts.h:292
CPLStrnlen
size_t CPLStrnlen(const char *pszStr, size_t nMaxLen)
Definition: cpl_string.cpp:2796
VSIVirtualHandle::Eof
virtual int Eof()=0
Test for end of file.
CSLDuplicate
char ** CSLDuplicate(CSLConstList papszStrList)
Definition: cpl_string.cpp:228
CPLString::Trim
CPLSTRING_METHOD_DLL CPLString & Trim()
Definition: cplstring.cpp:203
VSI_STAT_EXISTS_FLAG
#define VSI_STAT_EXISTS_FLAG
Definition: cpl_vsi.h:203
VSIMalloc
void * VSIMalloc(size_t)
Definition: cpl_vsisimple.cpp:558
CSLDestroy
void CSLDestroy(char **papszStrList)
Definition: cpl_string.cpp:200
VSIReadDirRecursive
char ** VSIReadDirRecursive(const char *pszPath)
Read names in a directory recursively.
Definition: cpl_vsil.cpp:160
GIntBig
long long GIntBig
Definition: cpl_port.h:248
VSIStatExL
int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
Get filesystem object info.
Definition: cpl_vsil.cpp:776
CPLString::vPrintf
CPLSTRING_METHOD_DLL CPLString & vPrintf(const char *pszFormat, va_list args)
Definition: cplstring.cpp:76
VSI_STAT_SET_ERROR_FLAG
#define VSI_STAT_SET_ERROR_FLAG
Definition: cpl_vsi.h:209
VSIDIR
struct VSIDIR VSIDIR
Definition: cpl_vsi.h:309
CPLStringList::Count
int Count() const
Definition: cplstringlist.cpp:218
VSISetCryptKey
void VSISetCryptKey(const GByte *pabyKey, int nKeySize)
Definition: cpl_vsil_crypt.cpp:205
CPLXMLNode::psNext
struct CPLXMLNode * psNext
Next sibling.
Definition: cpl_minixml.h:106
VSIGetDiskFreeSpace
GIntBig VSIGetDiskFreeSpace(const char *pszDirname)
Return free disk space available on the filesystem.
Definition: cpl_vsil.cpp:2318
CPL_FORMAT_STRING
#define CPL_FORMAT_STRING(arg)
Definition: cpl_port.h:929
VSIStatBufL
struct stat64 VSIStatBufL
Definition: cpl_vsi.h:194
cpl_port.h
VSIInstallOSSStreamingFileHandler
void VSIInstallOSSStreamingFileHandler(void)
Install /vsioss_streaming/ Alibaba Cloud Object Storage Service (OSS) file system handler (requires l...
Definition: cpl_vsil_curl_streaming.cpp:2158
VSIInstallSwiftStreamingFileHandler
void VSIInstallSwiftStreamingFileHandler(void)
Install /vsiswift_streamin/ OpenStack Swif Object Storage (Swift) file system handler (requires libcu...
Definition: cpl_vsil_curl_streaming.cpp:2177
EQUALN
#define EQUALN(a, b, n)
Definition: cpl_port.h:567
CPLStringList::StealList
char ** StealList()
Definition: cplstringlist.cpp:489
VSICurlPartialClearCache
void VSICurlPartialClearCache(const char *pszFilenamePrefix)
Clean local cache associated with /vsicurl/ (and related file systems) for a given filename (and its ...
Definition: cpl_vsil_curl.cpp:4378
VSIInstallS3StreamingFileHandler
void VSIInstallS3StreamingFileHandler(void)
Install /vsis3_streaming/ Amazon S3 file system handler (requires libcurl).
Definition: cpl_vsil_curl_streaming.cpp:2102
VSIVirtualHandle
Definition: cpl_vsi_virtual.h:56
VSIFOpenL
VSILFILE * VSIFOpenL(const char *, const char *)
Open file.
Definition: cpl_vsil.cpp:997
CPLE_None
#define CPLE_None
Definition: cpl_error.h:97
CSLAddString
char ** CSLAddString(char **papszStrList, const char *pszNewString)
Definition: cpl_string.cpp:83
VSIReadDir
char ** VSIReadDir(const char *)
Read names in a directory.
Definition: cpl_vsil.cpp:83
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Definition: cpl_vsi.h:142
VSIVirtualHandle::GetNativeFileDescriptor
virtual void * GetNativeFileDescriptor()
Returns the "native" file descriptor for the virtual handle.
Definition: cpl_vsi_virtual.h:101
CPLStrdup
char * CPLStrdup(const char *)
Definition: cpl_conv.cpp:293
CPLAtof
double CPLAtof(const char *)
Definition: cpl_strtod.cpp:117
CPLGetLastErrorType
CPLErr CPLGetLastErrorType(void)
Definition: cpl_error.cpp:842
VSICurlClearCache
void VSICurlClearCache(void)
Clean local cache associated with /vsicurl/ (and related file systems)
Definition: cpl_vsil_curl.cpp:4338
CPLE_IllegalArg
#define CPLE_IllegalArg
Definition: cpl_error.h:107
CPLScanUIntBig
GUIntBig CPLScanUIntBig(const char *, int)
Definition: cpl_conv.cpp:964
CPLXMLNode::psChild
struct CPLXMLNode * psChild
Child node.
Definition: cpl_minixml.h:118
VSIIngestFile
int VSIIngestFile(VSILFILE *fp, const char *pszFilename, GByte **ppabyRet, vsi_l_offset *pnSize, GIntBig nMaxSize)
Ingest a file into memory.
Definition: cpl_vsil.cpp:2044
VSIReadDirRecursiveTask
Definition: cpl_vsil.cpp:129
CPLQuietErrorHandler
void CPLQuietErrorHandler(CPLErr, CPLErrorNum, const char *)
Definition: cpl_error.cpp:967
VSIGetActualURL
const char * VSIGetActualURL(const char *pszFilename)
Returns the actual URL of a supplied filename.
Definition: cpl_vsil.cpp:912
CPLE_FileIO
#define CPLE_FileIO
Definition: cpl_error.h:103
CPLBase64DecodeInPlace
int CPLBase64DecodeInPlace(GByte *pszBase64)
Definition: cpl_base64.cpp:93
GUInt16
unsigned short GUInt16
Definition: cpl_port.h:213
CPLFree
#define CPLFree
Definition: cpl_conv.h:81
CPLGetExtension
const char * CPLGetExtension(const char *)
Definition: cpl_path.cpp:345
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Definition: cpl_port.h:997
CPLGetFilename
const char * CPLGetFilename(const char *)
Definition: cpl_path.cpp:260
VSI_STAT_SIZE_FLAG
#define VSI_STAT_SIZE_FLAG
Definition: cpl_vsi.h:207
VSIFGetNativeFileDescriptorL
void * VSIFGetNativeFileDescriptorL(VSILFILE *)
Returns the "native" file descriptor for the virtual handle.
Definition: cpl_vsil.cpp:2297
VSIFPrintfL
int VSIFPrintfL(VSILFILE *, const char *,...) EXPERIMENTAL_CPL_WARN_UNUSED_RESULT
Formatted write to file.
Definition: cpl_vsil.cpp:1918
CPLGetConfigOption
const char * CPLGetConfigOption(const char *, const char *)
Definition: cpl_conv.cpp:1692
VSILFILE
FILE VSILFILE
Definition: cpl_vsi.h:156
VSIVirtualHandle::Seek
virtual int Seek(vsi_l_offset nOffset, int nWhence)=0
Seek to requested offset.
STARTS_WITH_CI
#define STARTS_WITH_CI(a, b)
Definition: cpl_port.h:580
CPLE_AppDefined
#define CPLE_AppDefined
Definition: cpl_error.h:99

Generated for GDAL by doxygen 1.8.17.