| Line |
Branch |
Exec |
Source |
| 1 |
|
|
/** |
| 2 |
|
|
* This file is part of the CernVM File System. |
| 3 |
|
|
* |
| 4 |
|
|
* Read-only NFSv4.0 server (RFC 7530) on top of an inode-based Backend. |
| 5 |
|
|
* |
| 6 |
|
|
* The export is read-only and never grants delegations, share-deny conflicts |
| 7 |
|
|
* or locks, so open state cannot conflict between clients. This lets the |
| 8 |
|
|
* server keep all state in memory, never revoke leases, and grant reclaims |
| 9 |
|
|
* after a restart without a grace period. |
| 10 |
|
|
*/ |
| 11 |
|
|
|
| 12 |
|
|
#ifndef CVMFS_NFS4_NFS4_SERVER_H_ |
| 13 |
|
|
#define CVMFS_NFS4_NFS4_SERVER_H_ |
| 14 |
|
|
|
| 15 |
|
|
#include <stdint.h> |
| 16 |
|
|
|
| 17 |
|
|
#include <atomic> |
| 18 |
|
|
#include <condition_variable> |
| 19 |
|
|
#include <deque> |
| 20 |
|
|
#include <map> |
| 21 |
|
|
#include <memory> |
| 22 |
|
|
#include <set> |
| 23 |
|
|
#include <mutex> |
| 24 |
|
|
#include <string> |
| 25 |
|
|
#include <thread> |
| 26 |
|
|
#include <utility> |
| 27 |
|
|
#include <vector> |
| 28 |
|
|
|
| 29 |
|
|
#include "nfs4/backend.h" |
| 30 |
|
|
#include "nfs4/callback.h" |
| 31 |
|
|
#include "nfs4/filehandle.h" |
| 32 |
|
|
#include "nfs4/nfs4_const.h" |
| 33 |
|
|
#include "nfs4/rpc.h" |
| 34 |
|
|
#include "nfs4/xdr.h" |
| 35 |
|
|
|
| 36 |
|
|
namespace nfs4 { |
| 37 |
|
|
|
| 38 |
|
|
class Nfs4Server : public RpcProgram { |
| 39 |
|
|
public: |
| 40 |
|
|
struct Options { |
| 41 |
|
234 |
Options() |
| 42 |
|
234 |
: lease_time(90) |
| 43 |
|
234 |
, max_read(1u << 20) |
| 44 |
|
234 |
, max_compound_ops(128) |
| 45 |
|
234 |
, max_stateids(65536) |
| 46 |
|
234 |
, delay_after_ms(0) |
| 47 |
|
234 |
, executor_threads(32) |
| 48 |
|
234 |
, delegations(false) { } |
| 49 |
|
|
uint32_t lease_time; |
| 50 |
|
|
uint32_t max_read; |
| 51 |
|
|
uint32_t max_compound_ops; |
| 52 |
|
|
uint32_t max_stateids; |
| 53 |
|
|
/** |
| 54 |
|
|
* If non-zero, a COMPOUND that takes longer is answered with |
| 55 |
|
|
* NFS4ERR_DELAY and finishes in the background; the client retries. |
| 56 |
|
|
* Keeps soft mounts from timing out while the backend blocks, e.g. on a |
| 57 |
|
|
* large download. |
| 58 |
|
|
*/ |
| 59 |
|
|
uint32_t delay_after_ms; |
| 60 |
|
|
/** Threads running COMPOUNDs when delay_after_ms is set */ |
| 61 |
|
|
unsigned executor_threads; |
| 62 |
|
|
/** |
| 63 |
|
|
* Grant read delegations to clients with a working callback path, so |
| 64 |
|
|
* that they can cache without revalidating. All delegations are |
| 65 |
|
|
* recalled when the backend's generation changes. |
| 66 |
|
|
*/ |
| 67 |
|
|
bool delegations; |
| 68 |
|
|
}; |
| 69 |
|
|
|
| 70 |
|
|
Nfs4Server(Backend *backend, const Options &options); |
| 71 |
|
|
virtual ~Nfs4Server(); |
| 72 |
|
|
|
| 73 |
|
26 |
virtual uint32_t program() const { return kNfsProgram; } |
| 74 |
|
39 |
virtual uint32_t min_version() const { return kNfsVersion4; } |
| 75 |
|
26 |
virtual uint32_t max_version() const { return kNfsVersion4; } |
| 76 |
|
|
virtual AcceptStat Dispatch(const RpcCall &call, XdrReader *args, |
| 77 |
|
|
XdrWriter *res); |
| 78 |
|
|
|
| 79 |
|
|
/** Number of open stateids, exposed for tests */ |
| 80 |
|
|
size_t NumOpenStates(); |
| 81 |
|
|
/** Number of outstanding delegations, exposed for tests */ |
| 82 |
|
|
size_t NumDelegations(); |
| 83 |
|
|
|
| 84 |
|
|
private: |
| 85 |
|
|
struct Compound { |
| 86 |
|
1443 |
Compound() : call(NULL), has_fh(false), has_saved(false) { } |
| 87 |
|
|
const RpcCall *call; |
| 88 |
|
|
bool has_fh; |
| 89 |
|
|
FileHandle fh; |
| 90 |
|
|
bool has_saved; |
| 91 |
|
|
FileHandle saved; |
| 92 |
|
|
}; |
| 93 |
|
|
|
| 94 |
|
|
/** A resolved file handle: backend attributes plus what kind of object */ |
| 95 |
|
|
struct Object { |
| 96 |
|
|
FileHandle fh; |
| 97 |
|
|
Attr attr; |
| 98 |
|
|
std::string xattr_name; // set for named attributes |
| 99 |
|
|
}; |
| 100 |
|
|
|
| 101 |
|
|
struct Stateid { |
| 102 |
|
299 |
Stateid() : seqid(0) { } |
| 103 |
|
|
uint32_t seqid; |
| 104 |
|
|
std::string other; // kStateidOtherSize bytes |
| 105 |
|
|
}; |
| 106 |
|
|
|
| 107 |
|
|
struct Client { |
| 108 |
|
65 |
Client() : confirmed(false), callback_ok(false) { } |
| 109 |
|
|
std::string name; |
| 110 |
|
|
std::string verifier; |
| 111 |
|
|
std::string confirm; |
| 112 |
|
|
bool confirmed; |
| 113 |
|
|
std::shared_ptr<CallbackClient> callback; |
| 114 |
|
|
/** Set once CB_NULL succeeded; only then delegations are granted */ |
| 115 |
|
|
bool callback_ok; |
| 116 |
|
|
}; |
| 117 |
|
|
|
| 118 |
|
|
struct Delegation { |
| 119 |
|
13 |
Delegation() : clientid(0), seqid(1), recalled(0) { } |
| 120 |
|
|
uint64_t clientid; |
| 121 |
|
|
FileHandle fh; |
| 122 |
|
|
uint32_t seqid; |
| 123 |
|
|
/** When the recall was sent, 0 if not recalled */ |
| 124 |
|
|
time_t recalled; |
| 125 |
|
|
/** Attributes when the delegation was granted, see frozen_attrs_ */ |
| 126 |
|
|
Attr attr; |
| 127 |
|
|
}; |
| 128 |
|
|
|
| 129 |
|
|
typedef std::pair<uint64_t, std::string> OwnerKey; |
| 130 |
|
|
|
| 131 |
|
|
/** Open-owner sequencing state, including the reply for replays */ |
| 132 |
|
|
struct OpenOwner { |
| 133 |
|
52 |
OpenOwner() : seqid(0), last_op(0), last_status(0) { } |
| 134 |
|
|
uint32_t seqid; |
| 135 |
|
|
uint32_t last_op; |
| 136 |
|
|
uint32_t last_status; |
| 137 |
|
|
std::string last_body; |
| 138 |
|
|
}; |
| 139 |
|
|
|
| 140 |
|
|
struct OpenState { |
| 141 |
|
39 |
OpenState() : seqid(0), access(0) { } |
| 142 |
|
|
uint32_t seqid; |
| 143 |
|
|
OwnerKey owner; |
| 144 |
|
|
FileHandle fh; |
| 145 |
|
|
uint32_t access; |
| 146 |
|
|
}; |
| 147 |
|
|
|
| 148 |
|
|
enum SeqidCheck { |
| 149 |
|
|
kSeqidProcess, |
| 150 |
|
|
kSeqidReplay, |
| 151 |
|
|
kSeqidBad |
| 152 |
|
|
}; |
| 153 |
|
|
|
| 154 |
|
|
/** A COMPOUND handed to the executor threads */ |
| 155 |
|
|
struct Task { |
| 156 |
|
26 |
Task() : done(false) { } |
| 157 |
|
|
RpcCall call; |
| 158 |
|
|
std::string args; |
| 159 |
|
|
XdrWriter res; |
| 160 |
|
|
bool done; |
| 161 |
|
|
/** Signals done; guarded by executor_lock_ */ |
| 162 |
|
|
std::condition_variable done_cond; |
| 163 |
|
|
}; |
| 164 |
|
|
typedef std::shared_ptr<Task> TaskPtr; |
| 165 |
|
|
|
| 166 |
|
|
void RunWithDeadline(const RpcCall &call, XdrReader *args, XdrWriter *res); |
| 167 |
|
|
/** Probes callback paths and recalls delegations, see Options::delegations */ |
| 168 |
|
|
void MonitorLoop(); |
| 169 |
|
|
void StartMonitor(); |
| 170 |
|
|
uint32_t OpDelegreturn(XdrReader *args); |
| 171 |
|
|
/** Called with state_lock_ held; writes the open_delegation4 result */ |
| 172 |
|
|
void PutDelegation(uint64_t clientid, const FileHandle &fh, const Attr &attr, |
| 173 |
|
|
XdrWriter *w); |
| 174 |
|
|
/** Called with state_lock_ held, after delegations_ changed */ |
| 175 |
|
|
void UpdateFrozenAttrs(); |
| 176 |
|
|
void ExecutorLoop(); |
| 177 |
|
|
|
| 178 |
|
|
uint32_t ProcessCompound(const RpcCall &call, XdrReader *args, |
| 179 |
|
|
XdrWriter *res); |
| 180 |
|
|
uint32_t ProcessOp(uint32_t op, XdrReader *args, Compound *c, |
| 181 |
|
|
XdrWriter *body); |
| 182 |
|
|
|
| 183 |
|
|
uint32_t OpAccess(XdrReader *args, Compound *c, XdrWriter *body); |
| 184 |
|
|
uint32_t OpClose(XdrReader *args, Compound *c, XdrWriter *body); |
| 185 |
|
|
uint32_t OpGetattr(XdrReader *args, Compound *c, XdrWriter *body); |
| 186 |
|
|
uint32_t OpGetfh(Compound *c, XdrWriter *body); |
| 187 |
|
|
uint32_t OpLookup(XdrReader *args, Compound *c); |
| 188 |
|
|
uint32_t OpLookupp(Compound *c); |
| 189 |
|
|
uint32_t OpVerify(XdrReader *args, Compound *c, bool negate); |
| 190 |
|
|
uint32_t OpOpen(XdrReader *args, Compound *c, XdrWriter *body); |
| 191 |
|
|
uint32_t OpOpenattr(XdrReader *args, Compound *c); |
| 192 |
|
|
uint32_t OpOpenConfirm(XdrReader *args, Compound *c, XdrWriter *body); |
| 193 |
|
|
uint32_t OpOpenDowngrade(XdrReader *args, Compound *c, XdrWriter *body); |
| 194 |
|
|
uint32_t OpPutfh(XdrReader *args, Compound *c); |
| 195 |
|
|
uint32_t OpRead(XdrReader *args, Compound *c, XdrWriter *body); |
| 196 |
|
|
uint32_t OpReaddir(XdrReader *args, Compound *c, XdrWriter *body); |
| 197 |
|
|
uint32_t OpReadlink(Compound *c, XdrWriter *body); |
| 198 |
|
|
uint32_t OpRenew(XdrReader *args); |
| 199 |
|
|
uint32_t OpSecinfo(XdrReader *args, Compound *c, XdrWriter *body); |
| 200 |
|
|
uint32_t OpSetclientid(XdrReader *args, XdrWriter *body); |
| 201 |
|
|
uint32_t OpSetclientidConfirm(XdrReader *args); |
| 202 |
|
|
|
| 203 |
|
|
uint32_t Resolve(const FileHandle &fh, const Credentials &creds, Object *obj); |
| 204 |
|
|
uint32_t ResolveXattrName(uint64_t ino, uint32_t hash, std::string *name); |
| 205 |
|
|
uint32_t LookupChild(const Object &dir, const std::string &name, |
| 206 |
|
|
const Credentials &creds, Object *child); |
| 207 |
|
|
uint32_t ListChildren(const Object &dir, std::vector<DirEntry> *entries); |
| 208 |
|
|
|
| 209 |
|
|
bool EncodeFattr(const Object &obj, const uint32_t *request, |
| 210 |
|
|
unsigned request_len, const Credentials &creds, |
| 211 |
|
|
XdrWriter *out); |
| 212 |
|
|
static bool DecodeBitmap(XdrReader *r, std::vector<uint32_t> *bitmap); |
| 213 |
|
|
uint64_t FileId(const Object &obj) const; |
| 214 |
|
|
|
| 215 |
|
|
// State management, all called with state_lock_ held |
| 216 |
|
|
uint32_t CheckClient(uint64_t clientid); |
| 217 |
|
|
SeqidCheck CheckSeqid(const OwnerKey &owner, uint32_t seqid, uint32_t op, |
| 218 |
|
|
bool create_owner, uint32_t *status, std::string *body); |
| 219 |
|
|
void RecordSeqid(const OwnerKey &owner, uint32_t seqid, uint32_t op, |
| 220 |
|
|
uint32_t status, const std::string &body); |
| 221 |
|
|
uint32_t FindOpenState(const Stateid &stateid, bool exact_seqid, |
| 222 |
|
|
std::map<std::string, OpenState>::iterator *it); |
| 223 |
|
|
void DropClient(uint64_t clientid); |
| 224 |
|
|
std::string NewStateidOther(); |
| 225 |
|
|
/** For READ; accepts the anonymous and read-bypass special stateids */ |
| 226 |
|
|
uint32_t CheckReadStateid(const Stateid &stateid, const FileHandle &fh); |
| 227 |
|
|
|
| 228 |
|
|
static void PutStateid(const Stateid &stateid, XdrWriter *w); |
| 229 |
|
|
static bool GetStateid(XdrReader *r, Stateid *stateid); |
| 230 |
|
|
|
| 231 |
|
|
Backend *backend_; |
| 232 |
|
|
Options options_; |
| 233 |
|
|
uint64_t fsid_; |
| 234 |
|
|
/** Seconds since epoch at startup; prefixes client ids and stateids */ |
| 235 |
|
|
uint32_t boot_; |
| 236 |
|
|
|
| 237 |
|
|
std::mutex state_lock_; |
| 238 |
|
|
uint32_t next_clientid_; |
| 239 |
|
|
uint64_t next_stateid_; |
| 240 |
|
|
std::map<uint64_t, Client> clients_; |
| 241 |
|
|
std::map<OwnerKey, OpenOwner> owners_; |
| 242 |
|
|
std::map<std::string, OpenState> open_states_; |
| 243 |
|
|
std::map<std::string, Delegation> delegations_; |
| 244 |
|
|
/** |
| 245 |
|
|
* Files whose delegation of a client was recalled. The client's next open |
| 246 |
|
|
* gets no delegation: a client holding a delegation trusts its cache, so |
| 247 |
|
|
* it has to revalidate (close-to-open) before getting a new one. |
| 248 |
|
|
*/ |
| 249 |
|
|
std::set<std::pair<uint64_t, uint64_t> > recalled_files_; |
| 250 |
|
|
/** |
| 251 |
|
|
* While a recalled delegation is outstanding, its file reports the |
| 252 |
|
|
* attributes from when the delegation was granted. A client returns a |
| 253 |
|
|
* delegation with GETATTR + DELEGRETURN; if that GETATTR showed the new |
| 254 |
|
|
* attributes, the client (still holding the delegation) would take them as |
| 255 |
|
|
* its own and never notice the change. |
| 256 |
|
|
*/ |
| 257 |
|
|
std::map<uint64_t, Attr> frozen_attrs_; |
| 258 |
|
|
std::atomic<size_t> num_frozen_attrs_; |
| 259 |
|
|
std::vector<uint64_t> pending_probes_; |
| 260 |
|
|
uint64_t last_generation_; |
| 261 |
|
|
std::thread monitor_; |
| 262 |
|
|
bool monitor_stopping_; |
| 263 |
|
|
std::condition_variable monitor_cond_; |
| 264 |
|
|
|
| 265 |
|
|
// Executor threads start lazily, so that a server constructed before the |
| 266 |
|
|
// process daemonizes (forks) does not lose them |
| 267 |
|
|
std::mutex executor_lock_; |
| 268 |
|
|
/** Wakes executor threads; waiters use Task::done_cond */ |
| 269 |
|
|
std::condition_variable executor_cond_; |
| 270 |
|
|
std::deque<TaskPtr> tasks_; |
| 271 |
|
|
std::vector<std::thread> executors_; |
| 272 |
|
|
bool executors_stopping_; |
| 273 |
|
|
}; |
| 274 |
|
|
|
| 275 |
|
|
/** Maps a backend errno to an NFSv4 status */ |
| 276 |
|
|
uint32_t ErrnoToStatus(int err); |
| 277 |
|
|
|
| 278 |
|
|
} // namespace nfs4 |
| 279 |
|
|
|
| 280 |
|
|
#endif // CVMFS_NFS4_NFS4_SERVER_H_ |
| 281 |
|
|
|