XRootD
Loading...
Searching...
No Matches
XrdOssCsiPages Class Reference

#include <XrdOssCsiPages.hh>

+ Collaboration diagram for XrdOssCsiPages:

Public Types

typedef std::pair< off_t, off_t > Sizes_t
 

Public Member Functions

 XrdOssCsiPages (const std::string &fn, std::unique_ptr< XrdOssCsiTagstore > ts, bool wh, bool am, bool dpe, bool dlw, const char *)
 
 ~XrdOssCsiPages ()
 
void BasicConsistencyCheck (XrdOssDF *)
 
int Close ()
 
int FetchRange (XrdOssDF *, const void *, off_t, size_t, uint32_t *, uint64_t, XrdOssCsiRangeGuard &)
 
void Flush ()
 
int Fsync ()
 
bool IsReadOnly () const
 
int LockResetSizes (XrdOssDF *, off_t)
 
void LockTrackinglen (XrdOssCsiRangeGuard &, off_t, off_t, bool)
 
int Open (const char *path, off_t dsize, int flags, XrdOucEnv &envP)
 
int StoreRange (XrdOssDF *, const void *, off_t, size_t, uint32_t *, uint64_t, XrdOssCsiRangeGuard &)
 
void TrackedSizeRelease ()
 
int TrackedSizesGet (Sizes_t &, bool)
 
int truncate (XrdOssDF *, off_t, XrdOssCsiRangeGuard &)
 
int UpdateRange (XrdOssDF *, const void *, off_t, size_t, XrdOssCsiRangeGuard &)
 
int VerificationStatus ()
 
int VerifyRange (XrdOssDF *, const void *, off_t, size_t, XrdOssCsiRangeGuard &)
 

Static Public Member Functions

static void pgDoCalc (const void *, off_t, size_t, uint32_t *)
 
static int pgWritePrelockCheck (const void *, off_t, size_t, const uint32_t *, uint64_t)
 

Protected Member Functions

ssize_t apply_sequential_aligned_modify (const void *, off_t, size_t, const uint32_t *, bool, bool, uint32_t, uint32_t)
 
std::string ByteMismatchError (size_t blen, off_t off, uint8_t user, uint8_t page)
 
std::string CRCMismatchError (size_t blen, off_t pgnum, uint32_t got, uint32_t expected)
 
int FetchRangeAligned (const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
 
int FetchRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
 
int FetchRangeUnaligned_postblock (XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, size_t, uint64_t)
 
int FetchRangeUnaligned_preblock (XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, uint64_t)
 
int LockMakeUnverified ()
 
int LockSetTrackedSize (off_t)
 
int LockTruncateSize (off_t, bool)
 
std::string PageReadError (size_t blen, off_t pgnum, int ret)
 
int StoreRangeAligned (const void *, off_t, size_t, const Sizes_t &, uint32_t *)
 
int StoreRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, const uint32_t *)
 
int StoreRangeUnaligned_postblock (XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
 
int StoreRangeUnaligned_preblock (XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
 
std::string TagsReadError (off_t start, size_t n, int ret)
 
std::string TagsWriteError (off_t start, size_t n, int ret)
 
int UpdateRangeAligned (const void *, off_t, size_t, const Sizes_t &)
 
int UpdateRangeHoleUntilPage (XrdOssDF *, off_t, const Sizes_t &)
 
int UpdateRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
 
int VerifyRangeAligned (const void *, off_t, size_t, const Sizes_t &)
 
int VerifyRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
 

Static Protected Member Functions

static ssize_t fullread (XrdOssDF *fd, void *buff, const off_t off, const size_t sz)
 
static ssize_t maxread (XrdOssDF *fd, void *buff, const off_t off, const size_t sz, size_t tg=0)
 

Protected Attributes

bool allowMissingTags_
 
bool checklastpg_
 
bool disablePgExtend_
 
const std::string fn_
 
bool hasMissingTags_
 
off_t lastpgforloose_
 
bool loosewrite_
 
const bool loosewriteConfigured_
 
XrdSysMutex rangeaddmtx_
 
XrdOssCsiRanges ranges_
 
bool rdonly_
 
const char * tident
 
const std::string tident_
 
std::unique_ptr< XrdOssCsiTagstorets_
 
XrdSysCondVar tscond_
 
bool tsforupdate_
 
bool writeHoles_
 

Static Protected Attributes

static const size_t stsize_ = 1024
 

Detailed Description

Definition at line 46 of file XrdOssCsiPages.hh.

Member Typedef Documentation

◆ Sizes_t

std::pair<off_t,off_t> XrdOssCsiPages::Sizes_t

Definition at line 49 of file XrdOssCsiPages.hh.

Constructor & Destructor Documentation

◆ XrdOssCsiPages()

XrdOssCsiPages::XrdOssCsiPages ( const std::string & fn,
std::unique_ptr< XrdOssCsiTagstore > ts,
bool wh,
bool am,
bool dpe,
bool dlw,
const char * tid )

Definition at line 44 of file XrdOssCsiPages.cc.

44 :
45 ts_(std::move(ts)),
46 writeHoles_(wh),
49 hasMissingTags_(false),
50 rdonly_(false),
52 loosewrite_(false),
53 tscond_(0),
54 tsforupdate_(false),
55 fn_(fn),
56 tident_(tid),
57 tident(tident_.c_str()),
59 checklastpg_(false)
60{
61 // empty constructor
62}
static std::string ts()
timestamp output for logging messages
Definition XrdCephOss.cc:53
XrdSysCondVar tscond_
std::unique_ptr< XrdOssCsiTagstore > ts_
const char * tident
const std::string tident_
const bool loosewriteConfigured_
const std::string fn_

◆ ~XrdOssCsiPages()

XrdOssCsiPages::~XrdOssCsiPages ( )
inline

Definition at line 52 of file XrdOssCsiPages.hh.

52{ (void)Close(); }

References Close().

+ Here is the call graph for this function:

Member Function Documentation

◆ apply_sequential_aligned_modify()

ssize_t XrdOssCsiPages::apply_sequential_aligned_modify ( const void * buff,
off_t startp,
size_t nbytes,
const uint32_t * csvec,
bool preblockset,
bool lastblockset,
uint32_t cspre,
uint32_t cslast )
protected

Definition at line 288 of file XrdOssCsiPages.cc.

291{
292 EPNAME("apply_sequential_aligned_modify");
293
294 if (lastblockset && (nbytes % XrdSys::PageSize)==0)
295 {
296 return -EINVAL;
297 }
298 if (preblockset && startp==0)
299 {
300 return -EINVAL;
301 }
302
303 uint32_t calcbuf[stsize_];
304 const size_t calcbufsz = sizeof(calcbuf)/sizeof(uint32_t);
305 const uint8_t *const p = (uint8_t*)buff;
306
307 // will be using calcbuf
308 bool useinternal = true;
309 if (csvec && !preblockset && !lastblockset)
310 {
311 useinternal = false;
312 }
313
314 bool dopre = preblockset;
315 const off_t sp = preblockset ? startp-1 : startp;
316
317 size_t blktowrite = ((nbytes+XrdSys::PageSize-1)/XrdSys::PageSize) + (preblockset ? 1 : 0);
318 size_t nblkwritten = 0;
319 size_t calcbytot = 0;
320 while(blktowrite>0)
321 {
322 size_t blkwcnt = blktowrite;
323 if (useinternal)
324 {
325 size_t cidx = 0;
326 size_t calcbycnt = nbytes - calcbytot;
327 if (nblkwritten == 0 && dopre)
328 {
329 calcbycnt = std::min(calcbycnt, (calcbufsz-1)*XrdSys::PageSize);
330 blkwcnt = (calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize;
331 calcbuf[cidx] = cspre;
332 cidx++;
333 blkwcnt++;
334 dopre = false;
335 }
336 else
337 {
338 calcbycnt = std::min(calcbycnt, calcbufsz*XrdSys::PageSize);
339 blkwcnt = (calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize;
340 }
341 if ((calcbycnt % XrdSys::PageSize)!=0 && lastblockset)
342 {
343 const size_t x = calcbycnt / XrdSys::PageSize;
344 calcbycnt = XrdSys::PageSize * x;
345 calcbuf[cidx + x] = cslast;
346 }
347 if (csvec)
348 {
349 memcpy(&calcbuf[cidx], &csvec[calcbytot/XrdSys::PageSize], 4*((calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize));
350 }
351 else
352 {
353 XrdOucCRC::Calc32C(&p[calcbytot], calcbycnt, &calcbuf[cidx]);
354 }
355 calcbytot += calcbycnt;
356 }
357 const ssize_t wret = ts_->WriteTags(useinternal ? calcbuf : &csvec[nblkwritten], sp+nblkwritten, blkwcnt);
358 if (wret<0)
359 {
360 TRACE(Warn, TagsWriteError(sp+nblkwritten, blkwcnt, wret));
361 return wret;
362 }
363 blktowrite -= blkwcnt;
364 nblkwritten += blkwcnt;
365 }
366 return nblkwritten;
367}
#define EPNAME(x)
#define TRACE(act, x)
Definition XrdTrace.hh:63
std::string TagsWriteError(off_t start, size_t n, int ret)
static const size_t stsize_
virtual ssize_t WriteTags(const uint32_t *, off_t, size_t)=0
static uint32_t Calc32C(const void *data, size_t count, uint32_t prevcs=0)
Definition XrdOucCRC.cc:190
static const int PageSize

References XrdOucCRC::Calc32C(), EPNAME, XrdSys::PageSize, stsize_, TagsWriteError(), TRACE, ts_, and XrdOssCsiTagstore::WriteTags().

Referenced by StoreRangeAligned(), and StoreRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ BasicConsistencyCheck()

void XrdOssCsiPages::BasicConsistencyCheck ( XrdOssDF * fd)

Definition at line 820 of file XrdOssCsiPages.cc.

821{
822 EPNAME("BasicConsistencyCheck");
823
824 if (!loosewrite_ || rdonly_) return;
825
826 uint8_t b[XrdSys::PageSize];
827 static const uint8_t bz[XrdSys::PageSize] = {0};
828
829 const off_t tagsize = ts_->GetTrackedTagSize();
830 const off_t datasize = ts_->GetTrackedDataSize();
831
832 off_t taglp = 0, datalp = 0;
833 size_t tag_len = 0, data_len = 0;
834
835 if (tagsize>0)
836 {
837 taglp = (tagsize - 1) / XrdSys::PageSize;
838 tag_len = tagsize % XrdSys::PageSize;
839 tag_len = tag_len ? tag_len : XrdSys::PageSize;
840 }
841 if (datasize>0)
842 {
843 datalp = (datasize - 1) / XrdSys::PageSize;
844 data_len = datasize % XrdSys::PageSize;
845 data_len = data_len ? data_len : XrdSys::PageSize;
846 }
847
848 lastpgforloose_ = taglp;
849 checklastpg_ = true;
850
851 if (datasize>0 && taglp > datalp)
852 {
853 ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * datalp, XrdSys::PageSize);
854 if (rlen<0)
855 {
856 TRACE(Warn, PageReadError(XrdSys::PageSize, datalp, rlen));
857 return;
858 }
859
860 memset(&b[rlen], 0, XrdSys::PageSize-rlen);
861 const uint32_t data_crc = XrdOucCRC::Calc32C(b, data_len, 0u);
862 const uint32_t data_crc_z = XrdOucCRC::Calc32C(b, XrdSys::PageSize, 0u);
863 uint32_t tagv;
864 ssize_t rret = ts_->ReadTags(&tagv, datalp, 1);
865 if (rret<0)
866 {
867 TRACE(Warn, TagsReadError(datalp, 1, rret));
868 return;
869 }
870
871 if (tagv == data_crc_z)
872 {
873 // expected
874 }
875 else if (tagv == data_crc)
876 {
877 // should set tagv to data_crc_z
878 TRACE(Warn, "Resetting tag for page at " << datalp*XrdSys::PageSize << " to zero-extended");
879 const ssize_t wret = ts_->WriteTags(&data_crc_z, datalp, 1);
880 if (wret < 0)
881 {
882 TRACE(Warn, TagsWriteError(datalp, 1, wret));
883 return;
884 }
885 }
886 else
887 {
888 // something else wrong
889 TRACE(Warn, CRCMismatchError(data_len, datalp, data_crc, tagv) << " (ignoring)");
890 }
891 }
892 else if (tagsize>0 && taglp < datalp)
893 {
894 // datafile has more pages than recorded in the tag file:
895 // the tag file should have a crc corresponding to the relevant data fragment that is tracked in the last page.
896 // If it has the crc for a whole page (and there no non-zero content later in the page) reset it.
897 // This is so that a subsequnt UpdateRangeHoleUntilPage can zero-extend the CRC and get a consistent CRC.
898
899 ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * taglp, XrdSys::PageSize);
900 if (rlen<0)
901 {
902 TRACE(Warn, PageReadError(XrdSys::PageSize, taglp, rlen));
903 return;
904 }
905
906 memset(&b[rlen], 0, XrdSys::PageSize-rlen);
907 const uint32_t tag_crc = XrdOucCRC::Calc32C(b, tag_len, 0u);
908 const uint32_t tag_crc_z = XrdOucCRC::Calc32C(b, XrdSys::PageSize, 0u);
909 const uint32_t dp_ext_is_zero = !memcmp(&b[tag_len], bz, XrdSys::PageSize-tag_len);
910 uint32_t tagv;
911 ssize_t rret = ts_->ReadTags(&tagv, taglp, 1);
912 if (rret<0)
913 {
914 TRACE(Warn, TagsReadError(taglp, 1, rret));
915 return;
916 }
917
918 if (tagv == tag_crc)
919 {
920 // expected
921 }
922 else if (tagv == tag_crc_z && dp_ext_is_zero)
923 {
924 // should set tagv to tag_crc
925 TRACE(Warn, "Resetting tag for page at " << taglp*XrdSys::PageSize << " to not zero-extended");
926 const ssize_t wret = ts_->WriteTags(&tag_crc, taglp, 1);
927 if (wret < 0)
928 {
929 TRACE(Warn, TagsWriteError(taglp, 1, wret));
930 return;
931 }
932 }
933 else
934 {
935 // something else wrong
936 TRACE(Warn, CRCMismatchError(tag_len, taglp, tag_crc, tagv) << " dp_ext_is_zero=" << dp_ext_is_zero << " (ignoring)");
937 }
938 }
939}
static ssize_t maxread(XrdOssDF *fd, void *buff, const off_t off, const size_t sz, size_t tg=0)
std::string TagsReadError(off_t start, size_t n, int ret)
std::string CRCMismatchError(size_t blen, off_t pgnum, uint32_t got, uint32_t expected)
std::string PageReadError(size_t blen, off_t pgnum, int ret)
virtual off_t GetTrackedDataSize() const =0
virtual off_t GetTrackedTagSize() const =0
virtual ssize_t ReadTags(uint32_t *, off_t, size_t)=0

References XrdOucCRC::Calc32C(), checklastpg_, CRCMismatchError(), EPNAME, XrdOssCsiTagstore::GetTrackedDataSize(), XrdOssCsiTagstore::GetTrackedTagSize(), lastpgforloose_, loosewrite_, maxread(), PageReadError(), XrdSys::PageSize, rdonly_, XrdOssCsiTagstore::ReadTags(), TagsReadError(), TagsWriteError(), TRACE, ts_, and XrdOssCsiTagstore::WriteTags().

Referenced by LockResetSizes().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ ByteMismatchError()

std::string XrdOssCsiPages::ByteMismatchError ( size_t blen,
off_t off,
uint8_t user,
uint8_t page )
inlineprotected

Definition at line 161 of file XrdOssCsiPages.hh.

162 {
163 char buf[256],buf2[256];
164 snprintf(buf, sizeof(buf),
165 "unexpected byte mismatch between user-buffer and page/0x%04" PRIx32 " in file ",
166 (uint32_t)blen);
167 snprintf(buf2, sizeof(buf2),
168 " at offset 0x%" PRIx64 ", user-byte 0x%02" PRIx8 ", page-byte 0x%02" PRIx8,
169 (uint64_t)off,
170 user, page);
171 return buf + fn_ + buf2;
172 }

References fn_.

Referenced by FetchRangeUnaligned_postblock(), and FetchRangeUnaligned_preblock().

+ Here is the caller graph for this function:

◆ Close()

int XrdOssCsiPages::Close ( )

Definition at line 88 of file XrdOssCsiPages.cc.

89{
91 {
92 hasMissingTags_ = false;
93 return 0;
94 }
95 return ts_->Close();
96}
virtual int Close()=0

References XrdOssCsiTagstore::Close(), hasMissingTags_, and ts_.

Referenced by ~XrdOssCsiPages().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ CRCMismatchError()

std::string XrdOssCsiPages::CRCMismatchError ( size_t blen,
off_t pgnum,
uint32_t got,
uint32_t expected )
inlineprotected

Definition at line 148 of file XrdOssCsiPages.hh.

149 {
150 char buf[256],buf2[256];
151 snprintf(buf, sizeof(buf),
152 "bad crc32c/0x%04" PRIx32 " checksum in file ",
153 (uint32_t)blen);
154 snprintf(buf2, sizeof(buf2),
155 " at offset 0x%" PRIx64 ", got 0x%08" PRIx32 ", expected 0x%08" PRIx32,
156 (uint64_t)(pgnum*XrdSys::PageSize),
157 got, expected);
158 return buf + fn_ + buf2;
159 }

References fn_, and XrdSys::PageSize.

Referenced by BasicConsistencyCheck(), FetchRangeAligned(), FetchRangeUnaligned(), FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

+ Here is the caller graph for this function:

◆ FetchRange()

int XrdOssCsiPages::FetchRange ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
uint32_t * csvec,
uint64_t opts,
XrdOssCsiRangeGuard & rg )

Definition at line 627 of file XrdOssCsiPages.cc.

630{
631 EPNAME("FetchRange");
632 if (offset<0)
633 {
634 return -EINVAL;
635 }
636
637 // if the tag file is missing there is nothing to fetch or verify
638 // but if we should return a list of checksums calculate them from the data
639 if (hasMissingTags_)
640 {
641 if (csvec)
642 {
643 pgDoCalc(buff, offset, blen, csvec);
644 }
645 return 0;
646 }
647
648 const Sizes_t sizes = rg.getTrackinglens();
649 const off_t trackinglen = sizes.first;
650
651 if (offset >= trackinglen && blen == 0)
652 {
653 return 0;
654 }
655
656 if (blen == 0)
657 {
658 // if offset is before the tracked len we should not be requested to verify zero bytes:
659 // the file may have been truncated
660 TRACE(Warn, "Fetch request for zero bytes " << fn_ << ", file may be truncated");
661 return -EDOM;
662 }
663
664 if (offset+blen > static_cast<size_t>(trackinglen))
665 {
666 TRACE(Warn, "Fetch request for " << (offset+blen-trackinglen) << " bytes from " << fn_ << " beyond tracked length");
667 return -EDOM;
668 }
669
670 if (csvec == NULL && !(opts & XrdOssDF::Verify))
671 {
672 // if the crc values are not wanted nor checks against data, then
673 // there's nothing more to do here
674 return 0;
675 }
676
677 int fret;
678 if ((offset % XrdSys::PageSize) != 0 || (offset+blen != static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0))
679 {
680 fret = FetchRangeUnaligned(fd, buff, offset, blen, sizes, csvec, opts);
681 }
682 else
683 {
684 fret = FetchRangeAligned(buff,offset,blen,sizes,csvec,opts);
685 }
686 return fret;
687}
struct myOpts opts
int FetchRangeAligned(const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
std::pair< off_t, off_t > Sizes_t
int FetchRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
static void pgDoCalc(const void *, off_t, size_t, uint32_t *)
const std::pair< off_t, off_t > & getTrackinglens() const
static const uint64_t Verify
all: Verify checksums
Definition XrdOss.hh:223

References EPNAME, FetchRangeAligned(), FetchRangeUnaligned(), fn_, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, opts, XrdSys::PageSize, pgDoCalc(), TRACE, and XrdOssDF::Verify.

Referenced by XrdOssCsiFileAioJob::DoItRead2(), and XrdOssCsiFile::pgRead().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeAligned()

int XrdOssCsiPages::FetchRangeAligned ( const void * buff,
off_t offset,
size_t blen,
const Sizes_t & ,
uint32_t * csvec,
uint64_t opts )
protected

Definition at line 375 of file XrdOssCsiPages.cc.

376{
377 EPNAME("FetchRangeAligned");
378 uint32_t rdvec[stsize_],vrbuf[stsize_];
379
380 const off_t p1 = offset / XrdSys::PageSize;
381 const off_t p2 = (offset+blen) / XrdSys::PageSize;
382 const size_t p2_off = (offset+blen) % XrdSys::PageSize;
383 const size_t nfull = p2-p1;
384
385 uint32_t *rdbuf;
386 size_t rdbufsz;
387 if (csvec == NULL)
388 {
389 // use fixed sized stack buffer
390 rdbuf = rdvec;
391 rdbufsz = sizeof(rdvec)/sizeof(uint32_t);
392 }
393 else
394 {
395 // use supplied buffer, assumed to be large enough
396 rdbuf = csvec;
397 rdbufsz = (p2_off==0) ? nfull : (nfull+1);
398 }
399
400 // always use stack based, fixed sized buffer for verify
401 const size_t vrbufsz = sizeof(vrbuf)/sizeof(uint32_t);
402
403 // pointer to data
404 const uint8_t *const p = (uint8_t*)buff;
405
406 // process full pages + any partial page
407 size_t toread = (p2_off>0) ? nfull+1 : nfull;
408 size_t nread = 0;
409 while(toread>0)
410 {
411 const size_t rcnt = std::min(toread, rdbufsz-(nread%rdbufsz));
412 const ssize_t rret = ts_->ReadTags(&rdbuf[nread%rdbufsz], p1+nread, rcnt);
413 if (rret<0)
414 {
415 TRACE(Warn, TagsReadError(p1+nread, rcnt, rret));
416 return rret;
417 }
418 if ((opts & XrdOssDF::Verify))
419 {
420 size_t toverif = rcnt;
421 size_t nverif = 0;
422 while(toverif>0)
423 {
424 const size_t vcnt = std::min(toverif, vrbufsz);
425 const size_t databytes = (nread+nverif+vcnt <= nfull) ? (vcnt*XrdSys::PageSize) : ((vcnt-1)*XrdSys::PageSize+p2_off);
426 XrdOucCRC::Calc32C(&p[XrdSys::PageSize*(nread+nverif)],databytes,vrbuf);
427 if (memcmp(vrbuf, &rdbuf[(nread+nverif)%rdbufsz], 4*vcnt))
428 {
429 size_t badpg;
430 for(badpg=0;badpg<vcnt;++badpg) { if (memcmp(&vrbuf[badpg],&rdbuf[(nread+nverif+badpg)%rdbufsz],4)) break; }
431 TRACE(Warn, CRCMismatchError( (nread+nverif+badpg<nfull) ? XrdSys::PageSize : p2_off,
432 (p1+nread+nverif+badpg),
433 vrbuf[badpg],
434 rdbuf[(nread+nverif+badpg)%rdbufsz] ));
435 return -EDOM;
436 }
437 toverif -= vcnt;
438 nverif += vcnt;
439 }
440 }
441 toread -= rcnt;
442 nread += rcnt;
443 }
444
445 return 0;
446}

References XrdOucCRC::Calc32C(), CRCMismatchError(), EPNAME, opts, XrdSys::PageSize, XrdOssCsiTagstore::ReadTags(), stsize_, TagsReadError(), TRACE, ts_, and XrdOssDF::Verify.

Referenced by FetchRange(), and VerifyRangeAligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeUnaligned()

int XrdOssCsiPages::FetchRangeUnaligned ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes,
uint32_t * csvec,
uint64_t opts )
protected

Definition at line 803 of file XrdOssCsiPagesUnaligned.cc.

804{
805 EPNAME("FetchRangeUnaligned");
806
807 const off_t p1 = offset / XrdSys::PageSize;
808 const size_t p1_off = offset % XrdSys::PageSize;
809 const off_t p2 = (offset+blen) / XrdSys::PageSize;
810 const size_t p2_off = (offset+blen) % XrdSys::PageSize;
811
812 const off_t trackinglen = sizes.first;
813
814 size_t ntagstoread = (p2_off>0) ? p2-p1+1 : p2-p1;
815 size_t ntagsbase = p1;
816 uint32_t tbufint[stsize_], *tbuf=0;
817 size_t tbufsz = 0;
818 if (!csvec)
819 {
820 tbuf = tbufint;
821 tbufsz = sizeof(tbufint)/sizeof(uint32_t);
822 }
823 else
824 {
825 tbuf = csvec;
826 tbufsz = ntagstoread;
827 }
828
829 size_t tcnt = std::min(ntagstoread, tbufsz);
830 ssize_t rret = ts_->ReadTags(tbuf, ntagsbase, tcnt);
831 if (rret<0)
832 {
833 TRACE(Warn, TagsReadError(ntagsbase, tcnt, rret) << " (first)");
834 return rret;
835 }
836 ntagstoread -= tcnt;
837
838 // deal with partial first page
839 if ( p1_off>0 || blen < static_cast<size_t>(XrdSys::PageSize) )
840 {
841 const int ret = FetchRangeUnaligned_preblock(fd, buff, offset, blen, trackinglen, tbuf, csvec, opts);
842 if (ret<0)
843 {
844 return ret;
845 }
846 }
847
848 // first (inclusive) and last (exclusive) full page
849 const off_t fp = (p1_off != 0) ? p1+1 : p1;
850 const off_t lp = p2;
851
852 // verify full pages if wanted
853 if (fp<lp && (opts & XrdOssDF::Verify))
854 {
855 const uint8_t *const p = (uint8_t*)buff;
856 uint32_t calcbuf[stsize_];
857 const size_t cbufsz = sizeof(calcbuf)/sizeof(uint32_t);
858 size_t toread = lp-fp;
859 size_t nread = 0;
860 while(toread>0)
861 {
862 const size_t ccnt = std::min(toread, cbufsz);
863 XrdOucCRC::Calc32C(&p[(p1_off ? XrdSys::PageSize-p1_off : 0)+XrdSys::PageSize*nread],ccnt*XrdSys::PageSize,calcbuf);
864 size_t tovalid = ccnt;
865 size_t nvalid = 0;
866 while(tovalid>0)
867 {
868 const size_t tidx=fp+nread+nvalid - ntagsbase;
869 const size_t nv = std::min(tovalid, tbufsz-tidx);
870 if (nv == 0)
871 {
872 assert(csvec == NULL);
873 ntagsbase += tbufsz;
874 tcnt = std::min(ntagstoread, tbufsz);
875 rret = ts_->ReadTags(tbuf, ntagsbase, tcnt);
876 if (rret<0)
877 {
878 TRACE(Warn, TagsReadError(ntagsbase, tcnt, rret) << " (mid)");
879 return rret;
880 }
881 ntagstoread -= tcnt;
882 continue;
883 }
884 if (memcmp(&calcbuf[nvalid], &tbuf[tidx], 4*nv))
885 {
886 size_t badpg;
887 for(badpg=0;badpg<nv;badpg++) { if (memcmp(&calcbuf[nvalid+badpg], &tbuf[tidx+badpg],4)) break; }
889 (ntagsbase+tidx+badpg),
890 calcbuf[nvalid+badpg], tbuf[tidx+badpg]));
891 return -EDOM;
892 }
893 tovalid -= nv;
894 nvalid += nv;
895 }
896 toread -= ccnt;
897 nread += ccnt;
898 }
899 }
900
901 // last partial page
902 if (p2>p1 && p2_off > 0)
903 {
904 // make sure we have last tag;
905 // (should already have all of them if we're returning them in csvec)
906 size_t tidx = p2 - ntagsbase;
907 if (tidx >= tbufsz)
908 {
909 assert(csvec == NULL);
910 tidx = 0;
911 ntagsbase = p2;
912 rret = ts_->ReadTags(tbuf, ntagsbase, 1);
913 if (rret<0)
914 {
915 TRACE(Warn, TagsReadError(ntagsbase, 1, rret) << " (last)");
916 return rret;
917 }
918 ntagstoread = 0;
919 }
920
921 const int ret = FetchRangeUnaligned_postblock(fd, buff, offset, blen, trackinglen, tbuf, csvec, tidx, opts);
922 if (ret<0)
923 {
924 return ret;
925 }
926 }
927
928 return 0;
929}
int FetchRangeUnaligned_preblock(XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, uint64_t)
int FetchRangeUnaligned_postblock(XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, size_t, uint64_t)

References XrdOucCRC::Calc32C(), CRCMismatchError(), EPNAME, FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), opts, XrdSys::PageSize, XrdOssCsiTagstore::ReadTags(), stsize_, TagsReadError(), TRACE, ts_, and XrdOssDF::Verify.

Referenced by FetchRange(), and VerifyRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeUnaligned_postblock()

int XrdOssCsiPages::FetchRangeUnaligned_postblock ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
off_t trackinglen,
uint32_t * tbuf,
uint32_t * csvec,
size_t tidx,
uint64_t opts )
protected

Definition at line 727 of file XrdOssCsiPagesUnaligned.cc.

729{
730 EPNAME("FetchRangeUnaligned_postblock");
731
732 const off_t p2 = (offset+blen) / XrdSys::PageSize;
733 const size_t p2_off = (offset+blen) % XrdSys::PageSize;
734
735 // length of data in last (p2) page
736 const size_t bavail = std::min(trackinglen - (XrdSys::PageSize*p2), (off_t)XrdSys::PageSize);
737
738 // how much of that data is not being returned
739 const size_t bremain = (p2_off < bavail) ? bavail-p2_off : 0;
740 uint8_t b[XrdSys::PageSize];
741 const uint8_t *ub = &((uint8_t*)buff)[blen-p2_off];
742 if (bremain>0)
743 {
744 const ssize_t rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize*p2, bavail);
745 if (rret<0)
746 {
747 TRACE(Warn, PageReadError(bavail, p2, rret));
748 return rret;
749 }
750 // if we're verifying make sure overlapping part of data just read matches user's buffer
751 if ((opts & XrdOssDF::Verify))
752 {
753 const uint8_t *const p = (uint8_t*)buff;
754 if (memcmp(&p[blen-p2_off], b, p2_off))
755 {
756 size_t badoff;
757 for(badoff=0;badoff<p2_off;badoff++) { if (p[blen-p2_off+badoff] != b[badoff]) break; }
758 badoff = (badoff < p2_off) ? badoff : 0; // may be possible with concurrent modification
759 TRACE(Warn, ByteMismatchError(bavail, XrdSys::PageSize*p2+badoff, p[blen-p2_off+badoff], b[badoff]));
760 return -EDOM;
761 }
762 }
763 ub = b;
764 }
765 if ((opts & XrdOssDF::Verify))
766 {
767 const uint32_t crc32calc = XrdOucCRC::Calc32C(ub, bavail, 0U);
768 if (tbuf[tidx] != crc32calc)
769 {
770 TRACE(Warn, CRCMismatchError(bavail, p2, crc32calc, tbuf[tidx]));
771 return -EDOM;
772 }
773 }
774 // if we're returning csvec and user only request part of page
775 // adjust the crc
776 if (csvec && bremain>0)
777 {
778 if ((opts & XrdOssDF::Verify))
779 {
780 // verified; calculate crc based on common part of page.
781 csvec[tidx] = XrdOucCRC::Calc32C(b, p2_off, 0u);
782 }
783 else
784 {
785 // recalculate crc based on recorded checksum and adjusting for part of data not returned.
786 // If either the returned data or the data not included in the user's request are
787 // corrupt the returned checksum and returned data will (probably) mismatch.
788
789 const uint32_t crc32c = XrdOucCRC::Calc32C(&b[p2_off], bremain, 0u);
790 csvec[tidx] = CrcUtils.crc32c_split1(csvec[tidx], crc32c, bremain);
791 }
792 }
793
794 return 0;
795}
static XrdOssCsiCrcUtils CrcUtils
uint32_t crc32c(uint32_t crc, void const *buf, size_t len)
static uint32_t crc32c_split1(uint32_t crctot, uint32_t crc2, size_t len2)
std::string ByteMismatchError(size_t blen, off_t off, uint8_t user, uint8_t page)
static ssize_t fullread(XrdOssDF *fd, void *buff, const off_t off, const size_t sz)

References ByteMismatchError(), XrdOucCRC::Calc32C(), crc32c(), XrdOssCsiCrcUtils::crc32c_split1(), CRCMismatchError(), CrcUtils, EPNAME, fullread(), opts, PageReadError(), XrdSys::PageSize, TRACE, and XrdOssDF::Verify.

Referenced by FetchRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeUnaligned_preblock()

int XrdOssCsiPages::FetchRangeUnaligned_preblock ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
off_t trackinglen,
uint32_t * tbuf,
uint32_t * csvec,
uint64_t opts )
protected

Definition at line 639 of file XrdOssCsiPagesUnaligned.cc.

641{
642 EPNAME("FetchRangeUnaligned_preblock");
643
644 const off_t p1 = offset / XrdSys::PageSize;
645 const size_t p1_off = offset % XrdSys::PageSize;
646
647 // bavail is length of data in this page
648 const size_t bavail = std::min(trackinglen - (XrdSys::PageSize*p1), (off_t)XrdSys::PageSize);
649
650 // bcommon is length of data in this page that user wants
651 const size_t bcommon = std::min(bavail - p1_off, blen);
652
653 uint8_t b[XrdSys::PageSize];
654 const uint8_t *ub = (uint8_t*)buff;
655 if (bavail>bcommon)
656 {
657 // will need more data to either verify or return crc of the user's data
658 // (in case of no verify and no csvec FetchRange() returns early)
659 const ssize_t rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize*p1, bavail);
660 if (rret<0)
661 {
662 TRACE(Warn, PageReadError(bavail, p1, rret));
663 return rret;
664 }
665 // if we're going to verify, make sure we just read the same overlapping data as that in the user's buffer
666 if ((opts & XrdOssDF::Verify))
667 {
668 if (memcmp(buff, &b[p1_off], bcommon))
669 {
670 size_t badoff;
671 for(badoff=0;badoff<bcommon;badoff++) { if (((uint8_t*)buff)[badoff] != b[p1_off+badoff]) break; }
672 badoff = (badoff < bcommon) ? badoff : 0; // may be possible with concurrent modification
673 TRACE(Warn, ByteMismatchError(bavail, XrdSys::PageSize*p1+p1_off+badoff, ((uint8_t*)buff)[badoff], b[p1_off+badoff]));
674 return -EDOM;
675 }
676 }
677 ub = b;
678 }
679 // verify; based on whole block, or user's buffer (if it contains the whole block)
680 if ((opts & XrdOssDF::Verify))
681 {
682 const uint32_t crc32calc = XrdOucCRC::Calc32C(ub, bavail, 0U);
683 if (tbuf[0] != crc32calc)
684 {
685 TRACE(Warn, CRCMismatchError(bavail, p1, crc32calc, tbuf[0]));
686 return -EDOM;
687 }
688 }
689
690 // if we're returning csvec values and this first block
691 // needs adjustment because user requested a subset..
692 if (bavail>bcommon && csvec)
693 {
694 // make sure csvec[0] corresponds to only the data the user wanted, not whole page.
695 // if we have already verified the page + common part matches user's, take checksum of common.
696 // (Use local copy of page, perhaps less chance of accidental concurrent modification than buffer)
697 // Otherwise base on saved checksum.
698 if ((opts & XrdOssDF::Verify))
699 {
700 csvec[0] = XrdOucCRC::Calc32C(&b[p1_off], bcommon, 0u);
701 }
702 else
703 {
704 // calculate expected user checksum based on block's recorded checksum, adjusting
705 // for data not included in user's request. If either the returned data or the
706 // data not included in the user's request are corrupt the returned checksum and
707 // returned data will (probably) mismatch.
708
709 // remove block data before p1_off from checksum
710 uint32_t crc32c = XrdOucCRC::Calc32C(b, p1_off, 0u);
711 csvec[0] = CrcUtils.crc32c_split2(csvec[0], crc32c, bavail-p1_off);
712
713 // remove block data after p1_off+bcommon upto bavail
714 crc32c = XrdOucCRC::Calc32C(&b[p1_off+bcommon], bavail-p1_off-bcommon, 0u);
715 csvec[0] = CrcUtils.crc32c_split1(csvec[0], crc32c, bavail-p1_off-bcommon);
716 }
717 }
718 return 0;
719}
static uint32_t crc32c_split2(uint32_t crctot, uint32_t crc1, size_t len2)

References ByteMismatchError(), XrdOucCRC::Calc32C(), crc32c(), XrdOssCsiCrcUtils::crc32c_split1(), XrdOssCsiCrcUtils::crc32c_split2(), CRCMismatchError(), CrcUtils, EPNAME, fullread(), opts, PageReadError(), XrdSys::PageSize, TRACE, and XrdOssDF::Verify.

Referenced by FetchRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ Flush()

void XrdOssCsiPages::Flush ( )

Definition at line 98 of file XrdOssCsiPages.cc.

99{
100 if (!hasMissingTags_) ts_->Flush();
101}
virtual void Flush()=0

References XrdOssCsiTagstore::Flush(), hasMissingTags_, and ts_.

Referenced by XrdOssCsiFile::Flush().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ Fsync()

int XrdOssCsiPages::Fsync ( )

Definition at line 103 of file XrdOssCsiPages.cc.

104{
105 if (hasMissingTags_) return 0;
106 return ts_->Fsync();
107}
virtual int Fsync()=0

References XrdOssCsiTagstore::Fsync(), hasMissingTags_, and ts_.

Referenced by XrdOssCsiFile::Fsync().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ fullread()

static ssize_t XrdOssCsiPages::fullread ( XrdOssDF * fd,
void * buff,
const off_t off,
const size_t sz )
inlinestaticprotected

Definition at line 123 of file XrdOssCsiPages.hh.

124 {
125 ssize_t rret = maxread(fd, buff, off, sz);
126 if (rret<0) return rret;
127 if (static_cast<size_t>(rret) != sz) return -EDOM;
128 return rret;
129 }

References maxread().

Referenced by FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ IsReadOnly()

bool XrdOssCsiPages::IsReadOnly ( ) const
inline

Definition at line 68 of file XrdOssCsiPages.hh.

68{ return rdonly_; }

References rdonly_.

◆ LockMakeUnverified()

int XrdOssCsiPages::LockMakeUnverified ( )
protected

Definition at line 152 of file XrdOssCsiPages.cc.

153{
155 return ts_->SetUnverified();
156}
virtual int SetUnverified()=0

References XrdOssCsiTagstore::SetUnverified(), ts_, and tscond_.

Referenced by StoreRange(), and UpdateRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ LockResetSizes()

int XrdOssCsiPages::LockResetSizes ( XrdOssDF * fd,
off_t sz )

Definition at line 134 of file XrdOssCsiPages.cc.

135{
136 // nothing to do is no tag file
137 if (hasMissingTags_) return 0;
138
140 const int ret = ts_->ResetSizes(sz);
143 return ret;
144}
void BasicConsistencyCheck(XrdOssDF *)
virtual int ResetSizes(off_t)=0

References BasicConsistencyCheck(), hasMissingTags_, loosewrite_, loosewriteConfigured_, XrdOssCsiTagstore::ResetSizes(), ts_, and tscond_.

+ Here is the call graph for this function:

◆ LockSetTrackedSize()

int XrdOssCsiPages::LockSetTrackedSize ( off_t sz)
protected

Definition at line 128 of file XrdOssCsiPages.cc.

129{
131 return ts_->SetTrackedSize(sz);
132}
virtual int SetTrackedSize(off_t)=0

References XrdOssCsiTagstore::SetTrackedSize(), ts_, and tscond_.

Referenced by StoreRange(), and UpdateRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ LockTrackinglen()

void XrdOssCsiPages::LockTrackinglen ( XrdOssCsiRangeGuard & rg,
off_t offset,
off_t offend,
bool rdonly )

Definition at line 499 of file XrdOssCsiPages.cc.

500{
501 // no need to lock if we don't have a tag file
502 if (hasMissingTags_) return;
503
504 // in case of empty range the tracking len is not copied
505 if (offset == offend) return;
506
507 {
509
510 Sizes_t sizes;
511 (void)TrackedSizesGet(sizes, !rdonly);
512
513 // tag tracking data filesize, as recorded in the tagfile and for which the tagfile
514 // should be approprately sized, is sizes.first: usually the same as the in
515 // memory "actual" data filesize (sizes.second), but may differ after crashes or write failure.
516 const off_t trackinglen = sizes.first;
517
518 const off_t p1 = (offset>trackinglen ? trackinglen : offset) / XrdSys::PageSize;
519 bool unlock = false;
520 if (!rdonly && offend <= trackinglen)
521 {
522 unlock = true;
523 }
524
525 off_t p2 = offend / XrdSys::PageSize;
526 const size_t p2_off = offend % XrdSys::PageSize;
527
528 // range is exclusive
529 if (p2_off ==0) p2--;
530
531 ranges_.AddRange(p1, p2, rg, rdonly);
532
533 if (unlock)
534 {
536 }
537 rg.SetTrackingInfo(this, sizes, (!rdonly && !unlock));
538 }
539
540 rg.Wait();
541}
XrdOssCsiRanges ranges_
XrdSysMutex rangeaddmtx_
int TrackedSizesGet(Sizes_t &, bool)
void SetTrackingInfo(XrdOssCsiPages *p, const std::pair< off_t, off_t > &tsizes, bool locked)
void AddRange(const off_t start, const off_t end, XrdOssCsiRangeGuard &rg, bool rdonly)

References XrdOssCsiRanges::AddRange(), hasMissingTags_, XrdSys::PageSize, rangeaddmtx_, ranges_, XrdOssCsiRangeGuard::SetTrackingInfo(), TrackedSizeRelease(), TrackedSizesGet(), and XrdOssCsiRangeGuard::Wait().

Referenced by XrdOssCsiFileAioJob::DoItRead1(), XrdOssCsiFileAioJob::DoItWrite1(), XrdOssCsiFile::Ftruncate(), XrdOssCsiFile::pgRead(), XrdOssCsiFile::pgWrite(), XrdOssCsiFile::Read(), XrdOssCsiFile::ReadRaw(), XrdOssCsiFile::ReadV(), XrdOssCsiFile::Write(), and XrdOssCsiFile::WriteV().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ LockTruncateSize()

int XrdOssCsiPages::LockTruncateSize ( off_t sz,
bool datatoo )
protected

Definition at line 146 of file XrdOssCsiPages.cc.

147{
149 return ts_->Truncate(sz,datatoo);
150}
virtual int Truncate(off_t, bool)=0

References XrdOssCsiTagstore::Truncate(), ts_, and tscond_.

Referenced by truncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ maxread()

static ssize_t XrdOssCsiPages::maxread ( XrdOssDF * fd,
void * buff,
const off_t off,
const size_t sz,
size_t tg = 0 )
inlinestaticprotected

Definition at line 132 of file XrdOssCsiPages.hh.

133 {
134 size_t toread = sz, nread = 0;
135 uint8_t *p = (uint8_t*)buff;
136 tg = tg ? tg : sz;
137 while(toread>0 && nread<tg)
138 {
139 const ssize_t rret = fd->Read(&p[nread], off+nread, toread);
140 if (rret<0) return rret;
141 if (rret==0) break;
142 toread -= rret;
143 nread += rret;
144 }
145 return nread;
146 }
virtual ssize_t Read(off_t offset, size_t size)
Definition XrdOss.hh:281

References XrdOssDF::Read().

Referenced by BasicConsistencyCheck(), fullread(), and StoreRangeUnaligned_preblock().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ Open()

int XrdOssCsiPages::Open ( const char * path,
off_t dsize,
int flags,
XrdOucEnv & envP )

Definition at line 64 of file XrdOssCsiPages.cc.

65{
66 EPNAME("Pages::Open");
67 hasMissingTags_ = false;
68 rdonly_ = false;
69 int ret = ts_->Open(path, dsize, flags, envP);
70 if (ret == -ENOENT)
71 {
72 // no existing tag
74 {
75 TRACE(Info, "Opening with missing tagfile: " << fn_);
76 hasMissingTags_ = true;
77 return 0;
78 }
79 TRACE(Warn, "Could not open tagfile for " << fn_ << " error " << ret);
80 return -EDOM;
81 }
82 if (ret<0) return ret;
83 if ((flags & O_ACCMODE) == O_RDONLY) rdonly_ = true;
84 loosewrite_ = (dsize==0 && ts_->GetTrackedTagSize()==0) ? false : loosewriteConfigured_;
85 return 0;
86}
virtual int Open(const char *, off_t, int, XrdOucEnv &)=0

References allowMissingTags_, EPNAME, fn_, XrdOssCsiTagstore::GetTrackedTagSize(), hasMissingTags_, loosewrite_, loosewriteConfigured_, XrdOssCsiTagstore::Open(), rdonly_, TRACE, and ts_.

+ Here is the call graph for this function:

◆ PageReadError()

std::string XrdOssCsiPages::PageReadError ( size_t blen,
off_t pgnum,
int ret )
inlineprotected

Definition at line 174 of file XrdOssCsiPages.hh.

175 {
176 char buf[256],buf2[256];
177 snprintf(buf, sizeof(buf),
178 "error %d while reading page/0x%04" PRIx32 " in file ",
179 ret, (uint32_t)blen);
180 snprintf(buf2, sizeof(buf2),
181 " at offset 0x%" PRIx64,
182 (uint64_t)(pgnum*XrdSys::PageSize));
183 return buf + fn_ + buf2;
184 }

References fn_, and XrdSys::PageSize.

Referenced by BasicConsistencyCheck(), FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

+ Here is the caller graph for this function:

◆ pgDoCalc()

void XrdOssCsiPages::pgDoCalc ( const void * buffer,
off_t offset,
size_t wrlen,
uint32_t * csvec )
static

Definition at line 777 of file XrdOssCsiPages.cc.

778{
779 const size_t p_off = offset % XrdSys::PageSize;
780 const size_t p_alen = (p_off > 0) ? (XrdSys::PageSize - p_off) : wrlen;
781 if (p_alen < wrlen)
782 {
783 XrdOucCRC::Calc32C((uint8_t *)buffer+p_alen, wrlen-p_alen, &csvec[1]);
784 }
785 XrdOucCRC::Calc32C((void*)buffer, std::min(p_alen, wrlen), csvec);
786}

References XrdOucCRC::Calc32C(), and XrdSys::PageSize.

Referenced by FetchRange(), and StoreRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ pgWritePrelockCheck()

int XrdOssCsiPages::pgWritePrelockCheck ( const void * buffer,
off_t offset,
size_t wrlen,
const uint32_t * csvec,
uint64_t opts )
static

Definition at line 788 of file XrdOssCsiPages.cc.

789{
790 // do verify before taking locks to allow for faster fail
791 if (csvec && (opts & XrdOssDF::Verify))
792 {
793 uint32_t valcs;
794 const size_t p_off = offset % XrdSys::PageSize;
795 const size_t p_alen = (p_off > 0) ? (XrdSys::PageSize - p_off) : wrlen;
796 if (p_alen < wrlen)
797 {
798 if (XrdOucCRC::Ver32C((uint8_t *)buffer+p_alen, wrlen-p_alen, &csvec[1], valcs)>=0)
799 {
800 return -EDOM;
801 }
802 }
803 if (XrdOucCRC::Ver32C((void*)buffer, std::min(p_alen, wrlen), csvec, valcs)>=0)
804 {
805 return -EDOM;
806 }
807 }
808
809 return 0;
810}
static bool Ver32C(const void *data, size_t count, const uint32_t csval, uint32_t *csbad=0)
Definition XrdOucCRC.cc:222

References opts, XrdSys::PageSize, XrdOucCRC::Ver32C(), and XrdOssDF::Verify.

Referenced by XrdOssCsiFile::pgWrite(), and XrdOssCsiFile::pgWrite().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRange()

int XrdOssCsiPages::StoreRange ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
uint32_t * csvec,
uint64_t opts,
XrdOssCsiRangeGuard & rg )

Definition at line 691 of file XrdOssCsiPages.cc.

692{
693 if (offset<0)
694 {
695 return -EINVAL;
696 }
697
698 if (blen == 0)
699 {
700 return 0;
701 }
702
703 // if the tag file is missing there is nothing to store
704 // but do calculate checksums to return, if requested to do so
705 if (hasMissingTags_)
706 {
707 if (csvec && (opts & XrdOssDF::doCalc))
708 {
709 pgDoCalc(buff, offset, blen, csvec);
710 }
711 return 0;
712 }
713
714 const Sizes_t sizes = rg.getTrackinglens();
715 const off_t trackinglen = sizes.first;
716
717 // in the original specification of pgWrite there was the idea of a logical-eof, set by
718 // the a pgWrite with non-page aligned length: We support an option to approximate that
719 // by disallowing pgWrite past the current (non page aligned) eof.
720 if (disablePgExtend_ && (trackinglen % XrdSys::PageSize) !=0 && offset+blen > static_cast<size_t>(trackinglen))
721 {
722 return -ESPIPE;
723 }
724
725 // if doCalc is set and we have a csvec buffer fill it with calculated values
726 if (csvec && (opts & XrdOssDF::doCalc))
727 {
728 pgDoCalc(buff, offset, blen, csvec);
729 }
730
731 // if no vector of crc have been given and not specifically requested to calculate,
732 // then mark this file as having unverified checksums
733 if (!csvec && !(opts & XrdOssDF::doCalc))
734 {
736 }
737
738 if (offset+blen > static_cast<size_t>(trackinglen))
739 {
740 LockSetTrackedSize(offset+blen);
742 }
743
744 int ret;
745 if ((offset % XrdSys::PageSize) != 0 ||
746 (offset+blen < static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0) ||
747 ((trackinglen % XrdSys::PageSize) !=0 && offset > trackinglen))
748 {
749 ret = StoreRangeUnaligned(fd,buff,offset,blen,sizes,csvec);
750 }
751 else
752 {
753 ret = StoreRangeAligned(buff,offset,blen,sizes,csvec);
754 }
755
756 return ret;
757}
int StoreRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, const uint32_t *)
int StoreRangeAligned(const void *, off_t, size_t, const Sizes_t &, uint32_t *)
int LockSetTrackedSize(off_t)
static const uint64_t doCalc
pgw: Calculate checksums
Definition XrdOss.hh:225

References disablePgExtend_, XrdOssDF::doCalc, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockMakeUnverified(), LockSetTrackedSize(), opts, XrdSys::PageSize, pgDoCalc(), StoreRangeAligned(), StoreRangeUnaligned(), and XrdOssCsiRangeGuard::unlockTrackinglen().

Referenced by XrdOssCsiFileAioJob::DoItWrite1(), and XrdOssCsiFile::pgWrite().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeAligned()

int XrdOssCsiPages::StoreRangeAligned ( const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes,
uint32_t * csvec )
protected

Definition at line 453 of file XrdOssCsiPages.cc.

454{
455 EPNAME("StoreRangeAligned");
456
457 // if csvec given store those values
458 // if no csvec then calculate against data and store
459
460 const off_t p1 = offset / XrdSys::PageSize;
461 const off_t trackinglen = sizes.first;
462
463 if (offset > trackinglen)
464 {
465 const int ret = UpdateRangeHoleUntilPage(NULL, p1, sizes);
466 if (ret<0)
467 {
468 TRACE(Warn, "Error updating tags for holes, error=" << ret);
469 return ret;
470 }
471 }
472
473 const ssize_t aret = apply_sequential_aligned_modify(buff, p1, blen, csvec, false, false, 0U, 0U);
474 if (aret<0)
475 {
476 TRACE(Warn, "Error updating tags, error=" << aret);
477 return aret;
478 }
479
480 return 0;
481}
ssize_t apply_sequential_aligned_modify(const void *, off_t, size_t, const uint32_t *, bool, bool, uint32_t, uint32_t)
int UpdateRangeHoleUntilPage(XrdOssDF *, off_t, const Sizes_t &)

References apply_sequential_aligned_modify(), EPNAME, XrdSys::PageSize, TRACE, and UpdateRangeHoleUntilPage().

Referenced by StoreRange(), and UpdateRangeAligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeUnaligned()

int XrdOssCsiPages::StoreRangeUnaligned ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes,
const uint32_t * csvec )
protected

Definition at line 532 of file XrdOssCsiPagesUnaligned.cc.

533{
534 EPNAME("StoreRangeUnaligned");
535 const off_t p1 = offset / XrdSys::PageSize;
536
537 const off_t trackinglen = sizes.first;
538 if (offset > trackinglen)
539 {
540 const int ret = UpdateRangeHoleUntilPage(fd, p1, sizes);
541 if (ret<0)
542 {
543 TRACE(Warn, "Error updating tags for holes, error=" << ret);
544 return ret;
545 }
546 }
547
548 const size_t p1_off = offset % XrdSys::PageSize;
549 const size_t p2_off = (offset+blen) % XrdSys::PageSize;
550
551 bool hasprepage = false;
552 uint32_t prepageval;
553
554 // deal with partial first page
555 if ( p1_off>0 || blen < static_cast<size_t>(XrdSys::PageSize) )
556 {
557 const size_t bavail = (XrdSys::PageSize-p1_off > blen) ? blen : (XrdSys::PageSize-p1_off);
558 const int ret = StoreRangeUnaligned_preblock(fd, buff, bavail, offset, trackinglen, csvec, prepageval);
559 if (ret<0)
560 {
561 return ret;
562 }
563 hasprepage = true;
564 }
565
566 // next page (if any)
567 const off_t np = hasprepage ? p1+1 : p1;
568 // next page starts at buffer offset
569 const size_t npoff = hasprepage ? (XrdSys::PageSize - p1_off) : 0;
570
571 // anything in next page?
572 if (blen <= npoff)
573 {
574 // only need to write the first, partial page
575 if (hasprepage)
576 {
577 const ssize_t wret = ts_->WriteTags(&prepageval, p1, 1);
578 if (wret<0)
579 {
580 TRACE(Warn, TagsWriteError(p1, 1, wret));
581 return wret;
582 }
583 }
584 return 0;
585 }
586
587 const uint8_t *const p = (uint8_t*)buff;
588 const uint32_t *csp = csvec;
589 if (csp && hasprepage) csp++;
590
591 // see if there will be no old data to account for in the last page
592 if (p2_off == 0 || (offset + blen >= static_cast<size_t>(trackinglen)))
593 {
594 // write any precomputed prepage, then write full pages and last partial page (computing or using supplied csvec)
595 const ssize_t aret = apply_sequential_aligned_modify(&p[npoff], np, blen-npoff, csp, hasprepage, false, prepageval, 0U);
596 if (aret<0)
597 {
598 TRACE(Warn, "Error updating tags, error=" << aret);
599 return aret;
600 }
601 return 0;
602 }
603
604 // last page contains existing data that has to be read to modify it
605
606 uint32_t lastpageval;
607 const int ret = StoreRangeUnaligned_postblock(fd, &p[npoff], blen-npoff, offset+npoff, trackinglen, csp, lastpageval);
608 if (ret<0)
609 {
610 return ret;
611 }
612
613 // write any precomputed prepage, then write full pages (computing or using supplied csvec) and finally write precomputed last page
614 const ssize_t aret = apply_sequential_aligned_modify(&p[npoff], np, blen-npoff, csp, hasprepage, true, prepageval, lastpageval);
615 if (aret<0)
616 {
617 TRACE(Warn, "Error updating tags, error=" << aret);
618 return aret;
619 }
620
621 return 0;
622}
int StoreRangeUnaligned_preblock(XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
int StoreRangeUnaligned_postblock(XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)

References apply_sequential_aligned_modify(), EPNAME, XrdSys::PageSize, StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), TagsWriteError(), TRACE, ts_, UpdateRangeHoleUntilPage(), and XrdOssCsiTagstore::WriteTags().

Referenced by StoreRange(), and UpdateRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeUnaligned_postblock()

int XrdOssCsiPages::StoreRangeUnaligned_postblock ( XrdOssDF * fd,
const void * buff,
size_t blen,
off_t offset,
off_t trackinglen,
const uint32_t * csvec,
uint32_t & lastpageval )
protected

Definition at line 431 of file XrdOssCsiPagesUnaligned.cc.

434{
435 EPNAME("StoreRangeUnaligned_postblock");
436
437 const uint8_t *const p = (uint8_t*)buff;
438 const off_t p2 = (offset+blen) / XrdSys::PageSize;
439 const size_t p2_off = (offset+blen) % XrdSys::PageSize;
440
441 const off_t tracked_page = trackinglen / XrdSys::PageSize;
442 const size_t tracked_off = trackinglen % XrdSys::PageSize;
443
444 // we should not be called in this case
445 assert(p2_off != 0);
446
447 // how much existing data this last (p2) page
448 const size_t bavail = (p2==tracked_page) ? tracked_off : XrdSys::PageSize;
449
450 // how much of that data will not be overwritten
451 const size_t bremain = (p2_off < bavail) ? bavail-p2_off : 0;
452
453 // we should not be called if it is a complete overwrite
454 assert(bremain>0);
455
456 // need to use remaining data to calculate the crc of the new p2 page.
457 // read and verify it now.
458
459 uint32_t crc32v;
460 ssize_t rret = ts_->ReadTags(&crc32v, p2, 1);
461 if (rret<0)
462 {
463 TRACE(Warn, TagsReadError(p2, 1, rret));
464 return rret;
465 }
466
467 uint8_t b[XrdSys::PageSize];
468 rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize * p2, bavail);
469 if (rret<0)
470 {
471 TRACE(Warn, PageReadError(bavail, p2, rret));
472 return rret;
473 }
474
475 // calculate crc of new data with remaining data at the end:
476 uint32_t crc32c = 0;
477 if (csvec)
478 {
479 crc32c = csvec[(blen-1)/XrdSys::PageSize];
480 }
481 else
482 {
483 crc32c = XrdOucCRC::Calc32C(&p[blen-p2_off], p2_off, 0U);
484 }
485
486 const uint32_t cl = XrdOucCRC::Calc32C(&b[p2_off], bremain, 0U);
487 // crc of page with new data
488 crc32c = CrcUtils.crc32c_combine(crc32c, cl, bremain);
489 // crc of current page (before write)
490 const uint32_t crc32prev = XrdOucCRC::Calc32C(b, bavail, 0U);
491
492 // check(s) to see if remaining data was valid
493
494 // usual check; unmodified block is consistent with stored crc
495 // for loose write we allow case were the new crc has already been stored in the tagfile
496
497 // this may be an implicit verification (e.g. pgWrite may return EDOM without Verify requested)
498 // however, it's not clear if there is a meaningful way to crc a mismatching page during a partial overwrite
499 if (crc32v != crc32prev)
500 {
501 if (loosewrite_ && crc32c != crc32prev)
502 {
503 // log that we chceked if the tag was matching the previous data
504 TRACE(Warn, CRCMismatchError(bavail, p2, crc32prev, crc32v) << " (loose match, still trying)");
505 if (crc32c == crc32v)
506 {
507 TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p2) << " of file " << fn_);
508 lastpageval = crc32c;
509 return 0;
510 }
511 TRACE(Warn, CRCMismatchError(bavail, p2, crc32c, crc32v));
512 }
513 else
514 {
515 TRACE(Warn, CRCMismatchError(bavail, p2, crc32prev, crc32v));
516 }
517 return -EDOM;
518 }
519
520 lastpageval = crc32c;
521 return 0;
522}
static uint32_t crc32c_combine(uint32_t crc1, uint32_t crc2, size_t len2)

References XrdOucCRC::Calc32C(), crc32c(), XrdOssCsiCrcUtils::crc32c_combine(), CRCMismatchError(), CrcUtils, EPNAME, fn_, fullread(), loosewrite_, PageReadError(), XrdSys::PageSize, XrdOssCsiTagstore::ReadTags(), TagsReadError(), TRACE, and ts_.

Referenced by StoreRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeUnaligned_preblock()

int XrdOssCsiPages::StoreRangeUnaligned_preblock ( XrdOssDF * fd,
const void * buff,
size_t blen,
off_t offset,
off_t trackinglen,
const uint32_t * csvec,
uint32_t & prepageval )
protected

Definition at line 148 of file XrdOssCsiPagesUnaligned.cc.

151{
152 EPNAME("StoreRangeUnaligned_preblock");
153 const off_t p1 = offset / XrdSys::PageSize;
154 const size_t p1_off = offset % XrdSys::PageSize;
155
156 const off_t tracked_page = trackinglen / XrdSys::PageSize;
157 const size_t tracked_off = trackinglen % XrdSys::PageSize;
158
159 if (p1 > tracked_page)
160 {
161 // the start of will have a number of implied zero bytes
162 uint32_t crc32c = CrcUtils.crc32c_extendwith_zero(0u, p1_off);
163 if (csvec)
164 {
165 crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
166 }
167 else
168 {
169 crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
170 }
171 prepageval = crc32c;
172 return 0;
173 }
174
175 // we're appending, or appending within the last page after a gap of zeros
176 if (p1 == tracked_page && p1_off >= tracked_off)
177 {
178 // appending: with or without some implied zeros.
179
180 // zero initialised value may be used
181 uint32_t crc32v = 0;
182 if (tracked_off > 0)
183 {
184 const ssize_t rret = ts_->ReadTags(&crc32v, p1, 1);
185 if (rret<0)
186 {
187 TRACE(Warn, TagsReadError(p1, 1, rret) << " (append)");
188 return rret;
189 }
190 }
191
192 uint32_t crc32c = 0;
193
194 // only do the loosewrite extending check one time for the page which was the
195 // last page according to the trackinglen at time the check was configured (open or size-resync).
196 // don't do the check every time because it needs an extra read compared to the non loose case;
197 // checklastpg_ is checked and modified here, but is protected from concurrent
198 // access because of the condition that p1==lastpgforloose_
199
201 {
202 checklastpg_ = false;
203 uint8_t b[XrdSys::PageSize];
204
205 // this will reissue read() until eof, or tracked_off bytes read but accept up to PageSize
206 const ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * p1, XrdSys::PageSize, tracked_off);
207
208 if (rlen<0)
209 {
210 TRACE(Warn, PageReadError(tracked_off, p1, rlen));
211 return rlen;
212 }
213 memset(&b[rlen], 0, XrdSys::PageSize - rlen);
214
215 // in the loose-write mode, the new crc is based on the crc of data
216 // read from file up to p1_off, not on the previously stored tag.
217 // However must check if the data read were consistent with stored tag (crc32v)
218
219 uint32_t crc32x = XrdOucCRC::Calc32C(b, tracked_off, 0u);
220 crc32c = XrdOucCRC::Calc32C(&b[tracked_off], p1_off-tracked_off, crc32x);
221
222 do
223 {
224 if (static_cast<size_t>(rlen) == tracked_off)
225 {
226 // this is the expected match
227 if (tracked_off==0 || crc32x == crc32v) break;
228 }
229
230 // any bytes on disk beyond p1_off+blan would not be included in the new crc.
231 // if tracked_off==0 we have no meaningful crc32v value.
232 if ((tracked_off>0 || p1_off==0) && static_cast<size_t>(rlen) <= p1_off+blen)
233 {
234
235 if (tracked_off != 0)
236 {
237 TRACE(Warn, CRCMismatchError(tracked_off, p1, crc32x, crc32v) << " (loose match, still trying)");
238 }
239
240 // there was no tag recorded for the page, and we're completely overwriting anything on disk in the page
241 if (tracked_off==0)
242 {
243 TRACE(Warn, "Recovered page with no tag at offset " << (XrdSys::PageSize * p1) <<
244 " of file " << fn_ << " rlen=" << rlen << " (append)");
245 break;
246 }
247
248 if (static_cast<size_t>(rlen) != tracked_off && rlen>0)
249 {
250 crc32x = XrdOucCRC::Calc32C(b, rlen, 0u);
251 if (crc32x == crc32v)
252 {
253 TRACE(Warn, "Recovered page at offset " << (XrdSys::PageSize * p1)+p1_off << " of file " << fn_ << " (append)");
254 break;
255 }
256 TRACE(Warn, CRCMismatchError(rlen, p1, crc32x, crc32v) << " (loose match, still trying)");
257 }
258
259 memcpy(&b[p1_off], buff, blen);
260 crc32x = XrdOucCRC::Calc32C(b, p1_off+blen, 0u);
261 if (crc32x == crc32v)
262 {
263 TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p1)+p1_off <<
264 " of file " << fn_ << " (append)");
265 break;
266 }
267 TRACE(Warn, CRCMismatchError(p1_off+blen, p1, crc32x, crc32v) << " (append)");
268 }
269 else
270 {
271 if (tracked_off>0)
272 {
273 TRACE(Warn, CRCMismatchError(tracked_off, p1, crc32x, crc32v) << " (append)");
274 }
275 else
276 {
277 TRACE(Warn, "Unexpected content, write at page at offset " << (XrdSys::PageSize * p1) <<
278 " of file " << fn_ << ", offset-in-page=" << p1_off << " rlen=" << rlen << " (append)");
279 }
280 }
281 return -EDOM;
282 } while(0);
283 }
284 else
285 {
286 // non-loose case;
287 // can recalc crc with new data without re-reading existing partial block's data
288 const size_t nz = p1_off - tracked_off;
290 }
291
292 // crc32c is crc up to p1_off. Now add the user's data.
293 if (csvec)
294 {
295 crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
296 }
297 else
298 {
299 crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
300 }
301 prepageval = crc32c;
302 return 0;
303 }
304
305 const size_t bavail = (p1==tracked_page) ? tracked_off : XrdSys::PageSize;
306
307 // assert we're overwriting some (or all) of the previous data (other case was above)
308 assert(p1_off < bavail);
309
310 // case p1_off==0 && blen>=bavail is either handled by aligned case (p1==tracked_page)
311 // or not sent to preblock, so will need to read some preexisting data
312 assert(p1_off !=0 || blen<bavail);
313 uint8_t b[XrdSys::PageSize];
314
315 uint32_t crc32v;
316 ssize_t rret = ts_->ReadTags(&crc32v, p1, 1);
317 if (rret<0)
318 {
319 TRACE(Warn, TagsReadError(p1, 1, rret) << " (overwrite)");
320 return rret;
321 }
322
323 // in either loosewrite or non-loosewrite a read-modify-write sequence is done and the
324 // final crc is that of the modified block. The difference between loose and non-loose
325 // case if that the looser checks are done on the block.
326 //
327 // in either case there are implicit verification(s) (e.g. pgWrite may return EDOM without Verify requested)
328 // as it's not clear if there is a meaningful way to crc a mismatching page during a partial overwrite
329
330 if (loosewrite_)
331 {
332 // this will reissue read() until eof, or bavail bytes read but accept up to PageSize
333 const ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * p1, XrdSys::PageSize, bavail);
334 if (rlen<0)
335 {
336 TRACE(Warn, PageReadError(bavail, p1, rlen));
337 return rlen;
338 }
339 memset(&b[rlen], 0, XrdSys::PageSize - rlen);
340 do
341 {
342 uint32_t crc32c = XrdOucCRC::Calc32C(b, bavail, 0U);
343 // this is the expected case
344 if (static_cast<size_t>(rlen) == bavail && crc32c == crc32v) break;
345
346 // after this write there will be nothing changed between p1_off+blen
347 // and bavail; if there is nothing on disk in this range it will not
348 // be added by the write. So don't try to match crc with implied zero
349 // in this range. Beyond bavail bytes on disk will not be included
350 // in the new crc.
351 const size_t rmin = (p1_off+blen < bavail) ? bavail : 0;
352 if (static_cast<size_t>(rlen) >= rmin && static_cast<size_t>(rlen)<=bavail)
353 {
354 if (crc32c == crc32v)
355 {
356 TRACE(Warn, "Recovered page at offset " << (XrdSys::PageSize * p1) << " of file " << fn_ << " (overwrite)");
357 break;
358 }
359 TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v) << " (loose match, still trying)");
360
361 if (static_cast<size_t>(rlen) != bavail && rlen > 0)
362 {
363 crc32c = XrdOucCRC::Calc32C(b, rlen, 0U);
364 if (crc32c == crc32v)
365 {
366 TRACE(Warn, "Recovered page (2) at offset " << (XrdSys::PageSize * p1) << " of file " << fn_ << " (overwrite)");
367 break;
368 }
369 TRACE(Warn, CRCMismatchError(rlen, p1, crc32c, crc32v) << " (loose match, still trying)");
370 }
371
372 memcpy(&b[p1_off], buff, blen);
373 const size_t vl = std::max(bavail, p1_off+blen);
374 crc32c = XrdOucCRC::Calc32C(b, vl, 0U);
375 if (crc32c == crc32v)
376 {
377 TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p1)+p1_off << " of file " << fn_ << " (overwrite)");
378 break;
379 }
380 TRACE(Warn, CRCMismatchError(vl, p1, crc32c, crc32v) << " (overwrite)");
381 }
382 else
383 {
384 TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v) << " (overwrite)");
385 }
386 return -EDOM;
387 } while(0);
388 }
389 else
390 {
391 // non-loose case
392 rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize * p1, bavail);
393 if (rret<0)
394 {
395 TRACE(Warn, PageReadError(bavail, p1, rret));
396 return rret;
397 }
398 const uint32_t crc32c = XrdOucCRC::Calc32C(b, bavail, 0U);
399 if (crc32v != crc32c)
400 {
401 TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v));
402 return -EDOM;
403 }
404 }
405
406 uint32_t crc32c = XrdOucCRC::Calc32C(b, p1_off, 0U);
407 if (csvec)
408 {
409 crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
410 }
411 else
412 {
413 crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
414 }
415 if (p1_off+blen < bavail)
416 {
417 const uint32_t cl = XrdOucCRC::Calc32C(&b[p1_off+blen], bavail-p1_off-blen, 0U);
418 crc32c = CrcUtils.crc32c_combine(crc32c, cl, bavail-p1_off-blen);
419 }
420 prepageval = crc32c;
421 return 0;
422}
static uint32_t crc32c_extendwith_zero(uint32_t crc, size_t len)

References XrdOucCRC::Calc32C(), checklastpg_, crc32c(), XrdOssCsiCrcUtils::crc32c_combine(), XrdOssCsiCrcUtils::crc32c_extendwith_zero(), CRCMismatchError(), CrcUtils, EPNAME, fn_, fullread(), lastpgforloose_, loosewrite_, maxread(), PageReadError(), XrdSys::PageSize, XrdOssCsiTagstore::ReadTags(), TagsReadError(), TRACE, and ts_.

Referenced by StoreRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ TagsReadError()

std::string XrdOssCsiPages::TagsReadError ( off_t start,
size_t n,
int ret )
inlineprotected

Definition at line 186 of file XrdOssCsiPages.hh.

187 {
188 char buf[256];
189 snprintf(buf, sizeof(buf),
190 "error %d while reading crc32c values for pages [0x%" PRIx64":0x%" PRIx64 "] for file ",
191 ret, (uint64_t)start, (uint64_t)(start + n - 1));
192 return buf + fn_;
193 }

References fn_.

Referenced by BasicConsistencyCheck(), FetchRangeAligned(), FetchRangeUnaligned(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), truncate(), and UpdateRangeHoleUntilPage().

+ Here is the caller graph for this function:

◆ TagsWriteError()

std::string XrdOssCsiPages::TagsWriteError ( off_t start,
size_t n,
int ret )
inlineprotected

Definition at line 195 of file XrdOssCsiPages.hh.

196 {
197 char buf[256];
198 snprintf(buf, sizeof(buf),
199 "error %d while writing crc32c values for pages [0x%" PRIx64":0x%" PRIx64 "] for file ",
200 ret, (uint64_t)start, (uint64_t)(start + n - 1));
201 return buf + fn_;
202 }

References fn_.

Referenced by apply_sequential_aligned_modify(), BasicConsistencyCheck(), StoreRangeUnaligned(), truncate(), and UpdateRangeHoleUntilPage().

+ Here is the caller graph for this function:

◆ TrackedSizeRelease()

void XrdOssCsiPages::TrackedSizeRelease ( )

Definition at line 158 of file XrdOssCsiPages.cc.

159{
161 assert(tsforupdate_ == true);
162
163 tsforupdate_ = false;
165}

References XrdSysCondVar::Broadcast(), tscond_, and tsforupdate_.

Referenced by LockTrackinglen(), and XrdOssCsiRangeGuard::unlockTrackinglen().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ TrackedSizesGet()

int XrdOssCsiPages::TrackedSizesGet ( XrdOssCsiPages::Sizes_t & rsizes,
bool forupdate )

Definition at line 109 of file XrdOssCsiPages.cc.

110{
111 if (hasMissingTags_) return -ENOENT;
112
114 while (tsforupdate_)
115 {
116 tscond_.Wait();
117 }
118 off_t tagsize = ts_->GetTrackedTagSize();
119 off_t datasize = ts_->GetTrackedDataSize();
120 if (forupdate)
121 {
122 tsforupdate_ = true;
123 }
124 rsizes = std::make_pair(tagsize,datasize);
125 return 0;
126}

References XrdOssCsiTagstore::GetTrackedDataSize(), XrdOssCsiTagstore::GetTrackedTagSize(), hasMissingTags_, ts_, tscond_, tsforupdate_, and XrdSysCondVar::Wait().

Referenced by XrdOssCsiFile::Fstat(), and LockTrackinglen().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ truncate()

int XrdOssCsiPages::truncate ( XrdOssDF * fd,
off_t len,
XrdOssCsiRangeGuard & rg )

Definition at line 543 of file XrdOssCsiPages.cc.

544{
545 EPNAME("truncate");
546
547 if (len<0) return -EINVAL;
548
549 // nothing to truncate if there is no tag file
550 if (hasMissingTags_) return 0;
551
552 const Sizes_t sizes = rg.getTrackinglens();
553
554 const off_t trackinglen = sizes.first;
555 const off_t p_until = len / XrdSys::PageSize;
556 const size_t p_off = len % XrdSys::PageSize;
557
558 if (len>trackinglen)
559 {
560 int ret = UpdateRangeHoleUntilPage(fd,p_until,sizes);
561 if (ret<0)
562 {
563 TRACE(Warn, "Error updating tags for holes, error=" << ret);
564 return ret;
565 }
566 }
567
568 if (len != trackinglen && p_off != 0)
569 {
570 const off_t tracked_page = trackinglen / XrdSys::PageSize;
571 const size_t tracked_off = trackinglen % XrdSys::PageSize;
572 size_t toread = tracked_off;
573 if (len>trackinglen)
574 {
575 if (p_until != tracked_page) toread = 0;
576 }
577 else
578 {
579 if (p_until != tracked_page) toread = XrdSys::PageSize;
580 }
581 uint8_t b[XrdSys::PageSize];
582 if (toread>0)
583 {
584 ssize_t rret = XrdOssCsiPages::fullread(fd, b, p_until*XrdSys::PageSize, toread);
585 if (rret<0)
586 {
587 TRACE(Warn, PageReadError(toread, p_until, rret));
588 return rret;
589 }
590 const uint32_t crc32c = XrdOucCRC::Calc32C(b, toread, 0U);
591 uint32_t crc32v;
592 rret = ts_->ReadTags(&crc32v, p_until, 1);
593 if (rret<0)
594 {
595 TRACE(Warn, TagsReadError(p_until, 1, rret));
596 return rret;
597 }
598 if (crc32v != crc32c)
599 {
600 TRACE(Warn, CRCMismatchError(toread, p_until, crc32c, crc32v));
601 return -EDOM;
602 }
603 }
604 if (p_off > toread)
605 {
606 memset(&b[toread],0,p_off-toread);
607 }
608 const uint32_t crc32c = XrdOucCRC::Calc32C(b, p_off, 0U);
609 const ssize_t wret = ts_->WriteTags(&crc32c, p_until, 1);
610 if (wret < 0)
611 {
612 TRACE(Warn, TagsWriteError(p_until, 1, wret));
613 return wret;
614 }
615 }
616
617 LockTruncateSize(len,true);
619 return 0;
620}
int LockTruncateSize(off_t, bool)

References XrdOucCRC::Calc32C(), crc32c(), CRCMismatchError(), EPNAME, fullread(), XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockTruncateSize(), PageReadError(), XrdSys::PageSize, XrdOssCsiTagstore::ReadTags(), TagsReadError(), TagsWriteError(), TRACE, ts_, XrdOssCsiRangeGuard::unlockTrackinglen(), UpdateRangeHoleUntilPage(), and XrdOssCsiTagstore::WriteTags().

Referenced by XrdOssCsiFile::Ftruncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRange()

int XrdOssCsiPages::UpdateRange ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
XrdOssCsiRangeGuard & rg )

Definition at line 169 of file XrdOssCsiPages.cc.

170{
171 if (offset<0)
172 {
173 return -EINVAL;
174 }
175
176 if (blen == 0)
177 {
178 return 0;
179 }
180
181 // if the tag file is missing we don't need to store anything
182 if (hasMissingTags_)
183 {
184 return 0;
185 }
186
187 // update of file were checksums are based on the file data suppplied: as there's no separate
188 // source of checksum information mark this file as having unverified checksums
190
191 const Sizes_t sizes = rg.getTrackinglens();
192
193 const off_t trackinglen = sizes.first;
194 if (offset+blen > static_cast<size_t>(trackinglen))
195 {
196 LockSetTrackedSize(offset+blen);
198 }
199
200 int ret;
201 if ((offset % XrdSys::PageSize) != 0 ||
202 (offset+blen < static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0) ||
203 ((trackinglen % XrdSys::PageSize) !=0 && offset > trackinglen))
204 {
205 ret = UpdateRangeUnaligned(fd, buff, offset, blen, sizes);
206 }
207 else
208 {
209 ret = UpdateRangeAligned(buff, offset, blen, sizes);
210 }
211
212 return ret;
213}
int UpdateRangeAligned(const void *, off_t, size_t, const Sizes_t &)
int UpdateRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)

References XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockMakeUnverified(), LockSetTrackedSize(), XrdSys::PageSize, XrdOssCsiRangeGuard::unlockTrackinglen(), UpdateRangeAligned(), and UpdateRangeUnaligned().

Referenced by XrdOssCsiFileAioJob::DoItWrite1(), XrdOssCsiFile::Write(), and XrdOssCsiFile::WriteV().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRangeAligned()

int XrdOssCsiPages::UpdateRangeAligned ( const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes )
protected

Definition at line 485 of file XrdOssCsiPages.cc.

486{
487 return StoreRangeAligned(buff, offset, blen, sizes, NULL);
488}

References StoreRangeAligned().

Referenced by UpdateRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRangeHoleUntilPage()

int XrdOssCsiPages::UpdateRangeHoleUntilPage ( XrdOssDF * fd,
off_t until,
const Sizes_t & sizes )
protected

Definition at line 52 of file XrdOssCsiPagesUnaligned.cc.

53{
54 EPNAME("UpdateRangeHoleUntilPage");
55
56 static const uint32_t crczero = CrcUtils.crc32c_extendwith_zero(0u, XrdSys::PageSize);
57 static const std::vector<uint32_t> crc32Vec(stsize_, crczero);
58 static const std::vector<uint32_t> crc32VecZ(stsize_, 0u);
59
60 const off_t trackinglen = sizes.first;
61 const off_t tracked_page = trackinglen / XrdSys::PageSize;
62 if (until <= tracked_page) return 0;
63
64 const size_t tracked_off = trackinglen % XrdSys::PageSize;
65
66 // if last tracked page is before page "until" extend it
67 if (tracked_off>0)
68 {
69 if (fd == NULL)
70 {
71 TRACE(Warn, "Unexpected partially filled last page " << fn_);
72 return -EDOM;
73 }
74
75 uint32_t prevtag;
76 const ssize_t rret = ts_->ReadTags(&prevtag, tracked_page, 1);
77 if (rret < 0)
78 {
79 TRACE(Warn, TagsReadError(tracked_page, 1, rret));
80 return rret;
81 }
82
83 // extend prevtag up to PageSize. If there is a mismatch it will only be
84 // discovered during a later read (but this saves a read now).
85 const uint32_t crc32c = CrcUtils.crc32c_extendwith_zero(prevtag, XrdSys::PageSize - tracked_off);
86 const ssize_t wret = ts_->WriteTags(&crc32c, tracked_page, 1);
87 if (wret < 0)
88 {
89 TRACE(Warn, TagsWriteError(tracked_page, 1, wret) << " (prev)");
90 return wret;
91 }
92 }
93
94 const off_t nAllEmpty = (tracked_off>0) ? (until - tracked_page - 1) : (until - tracked_page);
95 const off_t firstEmpty = (tracked_off>0) ? (tracked_page + 1) : tracked_page;
96
97 off_t towrite = nAllEmpty;
98 off_t nwritten = 0;
99 while(towrite>0)
100 {
101 off_t wblks = 0;
102 const uint32_t *wpointer = 0;
103 // unless we are configured with nofill (!nofill means writeHoles_ is true), we fill
104 // in the crc for an implied zero block. Reads from this hole will be valid, which
105 // is the usual behavior for sparse files. If nofill is configured it is assumed that the
106 // user knows the application does not expect to make such reads, so we set a zero tag
107 // so there will be a mismatch to catch the situation; we write the zero tag explicitly
108 // so that the tag file itself is not sparse.
109 if (writeHoles_)
110 {
111 wblks = (off_t)crc32Vec.size();
112 wpointer = &crc32Vec[0];
113 }
114 else
115 {
116 wblks = (off_t)crc32VecZ.size();
117 wpointer = &crc32VecZ[0];
118 }
119 const size_t nw = std::min(towrite, wblks);
120 const ssize_t wret = ts_->WriteTags(wpointer, firstEmpty+nwritten, nw);
121 if (wret<0)
122 {
123 TRACE(Warn, TagsWriteError(firstEmpty+nwritten, nw, wret) << " (new)");
124 return wret;
125 }
126 towrite -= wret;
127 nwritten += wret;
128 }
129
130 return 0;
131}

References crc32c(), XrdOssCsiCrcUtils::crc32c_extendwith_zero(), CrcUtils, EPNAME, fn_, XrdSys::PageSize, XrdOssCsiTagstore::ReadTags(), stsize_, TagsReadError(), TagsWriteError(), TRACE, ts_, writeHoles_, and XrdOssCsiTagstore::WriteTags().

Referenced by StoreRangeAligned(), StoreRangeUnaligned(), and truncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRangeUnaligned()

int XrdOssCsiPages::UpdateRangeUnaligned ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes )
protected

Definition at line 137 of file XrdOssCsiPagesUnaligned.cc.

138{
139 return StoreRangeUnaligned(fd, buff, offset, blen, sizes, NULL);
140}

References StoreRangeUnaligned().

Referenced by UpdateRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerificationStatus()

int XrdOssCsiPages::VerificationStatus ( )

Definition at line 759 of file XrdOssCsiPages.cc.

760{
761 if (hasMissingTags_)
762 {
763 return 0;
764 }
765 bool iv;
766 {
768 iv = ts_->IsVerified();
769 }
770 if (iv)
771 {
772 return XrdOss::PF_csVer;
773 }
774 return XrdOss::PF_csVun;
775}
virtual bool IsVerified() const =0
static const int PF_csVer
verified file checksums present
Definition XrdOss.hh:778
static const int PF_csVun
unverified file checksums present
Definition XrdOss.hh:779

References hasMissingTags_, XrdOssCsiTagstore::IsVerified(), XrdOss::PF_csVer, XrdOss::PF_csVun, ts_, and tscond_.

Referenced by XrdOssCsiFile::VerificationStatus().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerifyRange()

int XrdOssCsiPages::VerifyRange ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
XrdOssCsiRangeGuard & rg )

Definition at line 220 of file XrdOssCsiPages.cc.

221{
222 EPNAME("VerifyRange");
223
224 if (offset<0)
225 {
226 return -EINVAL;
227 }
228
229 // if the tag file is missing we don't verify anything
230 if (hasMissingTags_)
231 {
232 return 0;
233 }
234
235 const Sizes_t sizes = rg.getTrackinglens();
236 const off_t trackinglen = sizes.first;
237
238 if (offset >= trackinglen && blen == 0)
239 {
240 return 0;
241 }
242
243 if (blen == 0)
244 {
245 // if offset is before the tracked len we should not be requested to verify zero bytes:
246 // the file may have been truncated
247 TRACE(Warn, "Verify request for zero bytes " << fn_ << ", file may be truncated");
248 return -EDOM;
249 }
250
251 if (offset+blen > static_cast<size_t>(trackinglen))
252 {
253 TRACE(Warn, "Verify request for " << (offset+blen-trackinglen) << " bytes from " << fn_ << " beyond tracked length");
254 return -EDOM;
255 }
256
257 int vret;
258 if ((offset % XrdSys::PageSize) != 0 || (offset+blen != static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0))
259 {
260 vret = VerifyRangeUnaligned(fd, buff, offset, blen, sizes);
261 }
262 else
263 {
264 vret = VerifyRangeAligned(buff, offset, blen, sizes);
265 }
266
267 return vret;
268}
int VerifyRangeAligned(const void *, off_t, size_t, const Sizes_t &)
int VerifyRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)

References EPNAME, fn_, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, XrdSys::PageSize, TRACE, VerifyRangeAligned(), and VerifyRangeUnaligned().

Referenced by XrdOssCsiFileAioJob::DoItRead2(), XrdOssCsiFile::Read(), XrdOssCsiFile::ReadRaw(), and XrdOssCsiFile::ReadV().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerifyRangeAligned()

int XrdOssCsiPages::VerifyRangeAligned ( const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes )
protected

Definition at line 448 of file XrdOssCsiPages.cc.

449{
450 return FetchRangeAligned(buff,offset,blen,sizes,NULL,XrdOssDF::Verify);
451}

References FetchRangeAligned(), and XrdOssDF::Verify.

Referenced by VerifyRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerifyRangeUnaligned()

int XrdOssCsiPages::VerifyRangeUnaligned ( XrdOssDF * fd,
const void * buff,
off_t offset,
size_t blen,
const Sizes_t & sizes )
protected

Definition at line 628 of file XrdOssCsiPagesUnaligned.cc.

629{
630 return FetchRangeUnaligned(fd, buff, offset, blen, sizes, NULL, XrdOssDF::Verify);
631}

References FetchRangeUnaligned(), and XrdOssDF::Verify.

Referenced by VerifyRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Member Data Documentation

◆ allowMissingTags_

bool XrdOssCsiPages::allowMissingTags_
protected

Definition at line 84 of file XrdOssCsiPages.hh.

Referenced by Open().

◆ checklastpg_

bool XrdOssCsiPages::checklastpg_
protected

Definition at line 102 of file XrdOssCsiPages.hh.

Referenced by BasicConsistencyCheck(), and StoreRangeUnaligned_preblock().

◆ disablePgExtend_

bool XrdOssCsiPages::disablePgExtend_
protected

Definition at line 85 of file XrdOssCsiPages.hh.

Referenced by StoreRange().

◆ fn_

◆ hasMissingTags_

bool XrdOssCsiPages::hasMissingTags_
protected

◆ lastpgforloose_

off_t XrdOssCsiPages::lastpgforloose_
protected

Definition at line 101 of file XrdOssCsiPages.hh.

Referenced by BasicConsistencyCheck(), and StoreRangeUnaligned_preblock().

◆ loosewrite_

bool XrdOssCsiPages::loosewrite_
protected

◆ loosewriteConfigured_

const bool XrdOssCsiPages::loosewriteConfigured_
protected

Definition at line 88 of file XrdOssCsiPages.hh.

Referenced by LockResetSizes(), and Open().

◆ rangeaddmtx_

XrdSysMutex XrdOssCsiPages::rangeaddmtx_
protected

Definition at line 81 of file XrdOssCsiPages.hh.

Referenced by LockTrackinglen().

◆ ranges_

XrdOssCsiRanges XrdOssCsiPages::ranges_
protected

Definition at line 82 of file XrdOssCsiPages.hh.

Referenced by LockTrackinglen().

◆ rdonly_

bool XrdOssCsiPages::rdonly_
protected

Definition at line 87 of file XrdOssCsiPages.hh.

Referenced by BasicConsistencyCheck(), IsReadOnly(), and Open().

◆ stsize_

const size_t XrdOssCsiPages::stsize_ = 1024
staticprotected

◆ tident

const char* XrdOssCsiPages::tident
protected

Definition at line 98 of file XrdOssCsiPages.hh.

◆ tident_

const std::string XrdOssCsiPages::tident_
protected

Definition at line 97 of file XrdOssCsiPages.hh.

◆ ts_

◆ tscond_

◆ tsforupdate_

bool XrdOssCsiPages::tsforupdate_
protected

Definition at line 92 of file XrdOssCsiPages.hh.

Referenced by TrackedSizeRelease(), and TrackedSizesGet().

◆ writeHoles_

bool XrdOssCsiPages::writeHoles_
protected

Definition at line 83 of file XrdOssCsiPages.hh.

Referenced by UpdateRangeHoleUntilPage().


The documentation for this class was generated from the following files: