module/zfs/dmu_send.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24
  25 #include <sys/dmu.h>
  26 #include <sys/dmu_impl.h>
  27 #include <sys/dmu_tx.h>
  28 #include <sys/dbuf.h>
  29 #include <sys/dnode.h>
  30 #include <sys/zfs_context.h>
  31 #include <sys/dmu_objset.h>
  32 #include <sys/dmu_traverse.h>
  33 #include <sys/dsl_dataset.h>
  34 #include <sys/dsl_dir.h>
  35 #include <sys/dsl_prop.h>
  36 #include <sys/dsl_pool.h>
  37 #include <sys/dsl_synctask.h>
  38 #include <sys/zfs_ioctl.h>
  39 #include <sys/zap.h>
  40 #include <sys/zio_checksum.h>
  41 #include <sys/zfs_znode.h>
  42 #include <zfs_fletcher.h>
  43 #include <sys/avl.h>
  44 #include <sys/ddt.h>
  45 #include <sys/zfs_onexit.h>
  46
  47 static char *dmu_recv_tag = "dmu_recv_tag";
  48
  49 /*
  50  * The list of data whose inclusion in a send stream can be pending from
  51  * one call to backup_cb to another.  Multiple calls to dump_free() and
  52  * dump_freeobjects() can be aggregated into a single DRR_FREE or
  53  * DRR_FREEOBJECTS replay record.
  54  */
  55 typedef enum {
  56         PENDING_NONE,
  57         PENDING_FREE,
  58         PENDING_FREEOBJECTS
  59 } pendop_t;
  60
  61 struct backuparg {
  62         dmu_replay_record_t *drr;
  63         vnode_t *vp;
  64         offset_t *off;
  65         objset_t *os;
  66         zio_cksum_t zc;
  67         uint64_t toguid;
  68         int err;
  69         pendop_t pending_op;
  70 };
  71
  72 static int
  73 dump_bytes(struct backuparg *ba, void *buf, int len)
  74 {
  75         ssize_t resid; /* have to get resid to get detailed errno */
  76         ASSERT3U(len % 8, ==, 0);
  77
  78         fletcher_4_incremental_native(buf, len, &ba->zc);
  79         ba->err = vn_rdwr(UIO_WRITE, ba->vp,
  80             (caddr_t)buf, len,
  81             0, UIO_SYSSPACE, FAPPEND, RLIM64_INFINITY, CRED(), &resid);
  82         *ba->off += len;
  83         return (ba->err);
  84 }
  85
  86 static int
  87 dump_free(struct backuparg *ba, uint64_t object, uint64_t offset,
  88     uint64_t length)
  89 {
  90         struct drr_free *drrf = &(ba->drr->drr_u.drr_free);
  91
  92         /*
  93          * If there is a pending op, but it's not PENDING_FREE, push it out,
  94          * since free block aggregation can only be done for blocks of the
  95          * same type (i.e., DRR_FREE records can only be aggregated with
  96          * other DRR_FREE records.  DRR_FREEOBJECTS records can only be
  97          * aggregated with other DRR_FREEOBJECTS records.
  98          */
  99         if (ba->pending_op != PENDING_NONE && ba->pending_op != PENDING_FREE) {
 100                 if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 101                         return (EINTR);
 102                 ba->pending_op = PENDING_NONE;
 103         }
 104
 105         if (ba->pending_op == PENDING_FREE) {
 106                 /*
 107                  * There should never be a PENDING_FREE if length is -1
 108                  * (because dump_dnode is the only place where this
 109                  * function is called with a -1, and only after flushing
 110                  * any pending record).
 111                  */
 112                 ASSERT(length != -1ULL);
 113                 /*
 114                  * Check to see whether this free block can be aggregated
 115                  * with pending one.
 116                  */
 117                 if (drrf->drr_object == object && drrf->drr_offset +
 118                     drrf->drr_length == offset) {
 119                         drrf->drr_length += length;
 120                         return (0);
 121                 } else {
 122                         /* not a continuation.  Push out pending record */
 123                         if (dump_bytes(ba, ba->drr,
 124                             sizeof (dmu_replay_record_t)) != 0)
 125                                 return (EINTR);
 126                         ba->pending_op = PENDING_NONE;
 127                 }
 128         }
 129         /* create a FREE record and make it pending */
 130         bzero(ba->drr, sizeof (dmu_replay_record_t));
 131         ba->drr->drr_type = DRR_FREE;
 132         drrf->drr_object = object;
 133         drrf->drr_offset = offset;
 134         drrf->drr_length = length;
 135         drrf->drr_toguid = ba->toguid;
 136         if (length == -1ULL) {
 137                 if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 138                         return (EINTR);
 139         } else {
 140                 ba->pending_op = PENDING_FREE;
 141         }
 142
 143         return (0);
 144 }
 145
 146 static int
 147 dump_data(struct backuparg *ba, dmu_object_type_t type,
 148     uint64_t object, uint64_t offset, int blksz, const blkptr_t *bp, void *data)
 149 {
 150         struct drr_write *drrw = &(ba->drr->drr_u.drr_write);
 151
 152
 153         /*
 154          * If there is any kind of pending aggregation (currently either
 155          * a grouping of free objects or free blocks), push it out to
 156          * the stream, since aggregation can't be done across operations
 157          * of different types.
 158          */
 159         if (ba->pending_op != PENDING_NONE) {
 160                 if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 161                         return (EINTR);
 162                 ba->pending_op = PENDING_NONE;
 163         }
 164         /* write a DATA record */
 165         bzero(ba->drr, sizeof (dmu_replay_record_t));
 166         ba->drr->drr_type = DRR_WRITE;
 167         drrw->drr_object = object;
 168         drrw->drr_type = type;
 169         drrw->drr_offset = offset;
 170         drrw->drr_length = blksz;
 171         drrw->drr_toguid = ba->toguid;
 172         drrw->drr_checksumtype = BP_GET_CHECKSUM(bp);
 173         if (zio_checksum_table[drrw->drr_checksumtype].ci_dedup)
 174                 drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP;
 175         DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp));
 176         DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp));
 177         DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp));
 178         drrw->drr_key.ddk_cksum = bp->blk_cksum;
 179
 180         if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 181                 return (EINTR);
 182         if (dump_bytes(ba, data, blksz) != 0)
 183                 return (EINTR);
 184         return (0);
 185 }
 186
 187 static int
 188 dump_spill(struct backuparg *ba, uint64_t object, int blksz, void *data)
 189 {
 190         struct drr_spill *drrs = &(ba->drr->drr_u.drr_spill);
 191
 192         if (ba->pending_op != PENDING_NONE) {
 193                 if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 194                         return (EINTR);
 195                 ba->pending_op = PENDING_NONE;
 196         }
 197
 198         /* write a SPILL record */
 199         bzero(ba->drr, sizeof (dmu_replay_record_t));
 200         ba->drr->drr_type = DRR_SPILL;
 201         drrs->drr_object = object;
 202         drrs->drr_length = blksz;
 203         drrs->drr_toguid = ba->toguid;
 204
 205         if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)))
 206                 return (EINTR);
 207         if (dump_bytes(ba, data, blksz))
 208                 return (EINTR);
 209         return (0);
 210 }
 211
 212 static int
 213 dump_freeobjects(struct backuparg *ba, uint64_t firstobj, uint64_t numobjs)
 214 {
 215         struct drr_freeobjects *drrfo = &(ba->drr->drr_u.drr_freeobjects);
 216
 217         /*
 218          * If there is a pending op, but it's not PENDING_FREEOBJECTS,
 219          * push it out, since free block aggregation can only be done for
 220          * blocks of the same type (i.e., DRR_FREE records can only be
 221          * aggregated with other DRR_FREE records.  DRR_FREEOBJECTS records
 222          * can only be aggregated with other DRR_FREEOBJECTS records.
 223          */
 224         if (ba->pending_op != PENDING_NONE &&
 225             ba->pending_op != PENDING_FREEOBJECTS) {
 226                 if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 227                         return (EINTR);
 228                 ba->pending_op = PENDING_NONE;
 229         }
 230         if (ba->pending_op == PENDING_FREEOBJECTS) {
 231                 /*
 232                  * See whether this free object array can be aggregated
 233                  * with pending one
 234                  */
 235                 if (drrfo->drr_firstobj + drrfo->drr_numobjs == firstobj) {
 236                         drrfo->drr_numobjs += numobjs;
 237                         return (0);
 238                 } else {
 239                         /* can't be aggregated.  Push out pending record */
 240                         if (dump_bytes(ba, ba->drr,
 241                             sizeof (dmu_replay_record_t)) != 0)
 242                                 return (EINTR);
 243                         ba->pending_op = PENDING_NONE;
 244                 }
 245         }
 246
 247         /* write a FREEOBJECTS record */
 248         bzero(ba->drr, sizeof (dmu_replay_record_t));
 249         ba->drr->drr_type = DRR_FREEOBJECTS;
 250         drrfo->drr_firstobj = firstobj;
 251         drrfo->drr_numobjs = numobjs;
 252         drrfo->drr_toguid = ba->toguid;
 253
 254         ba->pending_op = PENDING_FREEOBJECTS;
 255
 256         return (0);
 257 }
 258
 259 static int
 260 dump_dnode(struct backuparg *ba, uint64_t object, dnode_phys_t *dnp)
 261 {
 262         struct drr_object *drro = &(ba->drr->drr_u.drr_object);
 263
 264         if (dnp == NULL || dnp->dn_type == DMU_OT_NONE)
 265                 return (dump_freeobjects(ba, object, 1));
 266
 267         if (ba->pending_op != PENDING_NONE) {
 268                 if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 269                         return (EINTR);
 270                 ba->pending_op = PENDING_NONE;
 271         }
 272
 273         /* write an OBJECT record */
 274         bzero(ba->drr, sizeof (dmu_replay_record_t));
 275         ba->drr->drr_type = DRR_OBJECT;
 276         drro->drr_object = object;
 277         drro->drr_type = dnp->dn_type;
 278         drro->drr_bonustype = dnp->dn_bonustype;
 279         drro->drr_blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
 280         drro->drr_bonuslen = dnp->dn_bonuslen;
 281         drro->drr_checksumtype = dnp->dn_checksum;
 282         drro->drr_compress = dnp->dn_compress;
 283         drro->drr_toguid = ba->toguid;
 284
 285         if (dump_bytes(ba, ba->drr, sizeof (dmu_replay_record_t)) != 0)
 286                 return (EINTR);
 287
 288         if (dump_bytes(ba, DN_BONUS(dnp), P2ROUNDUP(dnp->dn_bonuslen, 8)) != 0)
 289                 return (EINTR);
 290
 291         /* free anything past the end of the file */
 292         if (dump_free(ba, object, (dnp->dn_maxblkid + 1) *
 293             (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT), -1ULL))
 294                 return (EINTR);
 295         if (ba->err)
 296                 return (EINTR);
 297         return (0);
 298 }
 299
 300 #define BP_SPAN(dnp, level) \
 301         (((uint64_t)dnp->dn_datablkszsec) << (SPA_MINBLOCKSHIFT + \
 302         (level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)))
 303
 304 /* ARGSUSED */
 305 static int
 306 backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, arc_buf_t *pbuf,
 307     const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
 308 {
 309         struct backuparg *ba = arg;
 310         dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE;
 311         int err = 0;
 312
 313         if (issig(JUSTLOOKING) && issig(FORREAL))
 314                 return (EINTR);
 315
 316         if (zb->zb_object != DMU_META_DNODE_OBJECT &&
 317             DMU_OBJECT_IS_SPECIAL(zb->zb_object)) {
 318                 return (0);
 319         } else if (bp == NULL && zb->zb_object == DMU_META_DNODE_OBJECT) {
 320                 uint64_t span = BP_SPAN(dnp, zb->zb_level);
 321                 uint64_t dnobj = (zb->zb_blkid * span) >> DNODE_SHIFT;
 322                 err = dump_freeobjects(ba, dnobj, span >> DNODE_SHIFT);
 323         } else if (bp == NULL) {
 324                 uint64_t span = BP_SPAN(dnp, zb->zb_level);
 325                 err = dump_free(ba, zb->zb_object, zb->zb_blkid * span, span);
 326         } else if (zb->zb_level > 0 || type == DMU_OT_OBJSET) {
 327                 return (0);
 328         } else if (type == DMU_OT_DNODE) {
 329                 dnode_phys_t *blk;
 330                 int i;
 331                 int blksz = BP_GET_LSIZE(bp);
 332                 uint32_t aflags = ARC_WAIT;
 333                 arc_buf_t *abuf;
 334
 335                 if (dsl_read(NULL, spa, bp, pbuf,
 336                     arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 337                     ZIO_FLAG_CANFAIL, &aflags, zb) != 0)
 338                         return (EIO);
 339
 340                 blk = abuf->b_data;
 341                 for (i = 0; i < blksz >> DNODE_SHIFT; i++) {
 342                         uint64_t dnobj = (zb->zb_blkid <<
 343                             (DNODE_BLOCK_SHIFT - DNODE_SHIFT)) + i;
 344                         err = dump_dnode(ba, dnobj, blk+i);
 345                         if (err)
 346                                 break;
 347                 }
 348                 (void) arc_buf_remove_ref(abuf, &abuf);
 349         } else if (type == DMU_OT_SA) {
 350                 uint32_t aflags = ARC_WAIT;
 351                 arc_buf_t *abuf;
 352                 int blksz = BP_GET_LSIZE(bp);
 353
 354                 if (arc_read_nolock(NULL, spa, bp,
 355                     arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 356                     ZIO_FLAG_CANFAIL, &aflags, zb) != 0)
 357                         return (EIO);
 358
 359                 err = dump_spill(ba, zb->zb_object, blksz, abuf->b_data);
 360                 (void) arc_buf_remove_ref(abuf, &abuf);
 361         } else { /* it's a level-0 block of a regular object */
 362                 uint32_t aflags = ARC_WAIT;
 363                 arc_buf_t *abuf;
 364                 int blksz = BP_GET_LSIZE(bp);
 365
 366                 if (dsl_read(NULL, spa, bp, pbuf,
 367                     arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 368                     ZIO_FLAG_CANFAIL, &aflags, zb) != 0)
 369                         return (EIO);
 370
 371                 err = dump_data(ba, type, zb->zb_object, zb->zb_blkid * blksz,
 372                     blksz, bp, abuf->b_data);
 373                 (void) arc_buf_remove_ref(abuf, &abuf);
 374         }
 375
 376         ASSERT(err == 0 || err == EINTR);
 377         return (err);
 378 }
 379
 380 int
 381 dmu_sendbackup(objset_t *tosnap, objset_t *fromsnap, boolean_t fromorigin,
 382     vnode_t *vp, offset_t *off)
 383 {
 384         dsl_dataset_t *ds = tosnap->os_dsl_dataset;
 385         dsl_dataset_t *fromds = fromsnap ? fromsnap->os_dsl_dataset : NULL;
 386         dmu_replay_record_t *drr;
 387         struct backuparg ba;
 388         int err;
 389         uint64_t fromtxg = 0;
 390
 391         /* tosnap must be a snapshot */
 392         if (ds->ds_phys->ds_next_snap_obj == 0)
 393                 return (EINVAL);
 394
 395         /* fromsnap must be an earlier snapshot from the same fs as tosnap */
 396         if (fromds && (ds->ds_dir != fromds->ds_dir ||
 397             fromds->ds_phys->ds_creation_txg >= ds->ds_phys->ds_creation_txg))
 398                 return (EXDEV);
 399
 400         if (fromorigin) {
 401                 dsl_pool_t *dp = ds->ds_dir->dd_pool;
 402
 403                 if (fromsnap)
 404                         return (EINVAL);
 405
 406                 if (dsl_dir_is_clone(ds->ds_dir)) {
 407                         rw_enter(&dp->dp_config_rwlock, RW_READER);
 408                         err = dsl_dataset_hold_obj(dp,
 409                             ds->ds_dir->dd_phys->dd_origin_obj, FTAG, &fromds);
 410                         rw_exit(&dp->dp_config_rwlock);
 411                         if (err)
 412                                 return (err);
 413                 } else {
 414                         fromorigin = B_FALSE;
 415                 }
 416         }
 417
 418
 419         drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
 420         drr->drr_type = DRR_BEGIN;
 421         drr->drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
 422         DMU_SET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo,
 423             DMU_SUBSTREAM);
 424
 425 #ifdef _KERNEL
 426         if (dmu_objset_type(tosnap) == DMU_OST_ZFS) {
 427                 uint64_t version;
 428                 if (zfs_get_zplprop(tosnap, ZFS_PROP_VERSION, &version) != 0)
 429                         return (EINVAL);
 430                 if (version == ZPL_VERSION_SA) {
 431                         DMU_SET_FEATUREFLAGS(
 432                             drr->drr_u.drr_begin.drr_versioninfo,
 433                             DMU_BACKUP_FEATURE_SA_SPILL);
 434                 }
 435         }
 436 #endif
 437
 438         drr->drr_u.drr_begin.drr_creation_time =
 439             ds->ds_phys->ds_creation_time;
 440         drr->drr_u.drr_begin.drr_type = tosnap->os_phys->os_type;
 441         if (fromorigin)
 442                 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CLONE;
 443         drr->drr_u.drr_begin.drr_toguid = ds->ds_phys->ds_guid;
 444         if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
 445                 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA;
 446
 447         if (fromds)
 448                 drr->drr_u.drr_begin.drr_fromguid = fromds->ds_phys->ds_guid;
 449         dsl_dataset_name(ds, drr->drr_u.drr_begin.drr_toname);
 450
 451         if (fromds)
 452                 fromtxg = fromds->ds_phys->ds_creation_txg;
 453         if (fromorigin)
 454                 dsl_dataset_rele(fromds, FTAG);
 455
 456         ba.drr = drr;
 457         ba.vp = vp;
 458         ba.os = tosnap;
 459         ba.off = off;
 460         ba.toguid = ds->ds_phys->ds_guid;
 461         ZIO_SET_CHECKSUM(&ba.zc, 0, 0, 0, 0);
 462         ba.pending_op = PENDING_NONE;
 463
 464         if (dump_bytes(&ba, drr, sizeof (dmu_replay_record_t)) != 0) {
 465                 kmem_free(drr, sizeof (dmu_replay_record_t));
 466                 return (ba.err);
 467         }
 468
 469         err = traverse_dataset(ds, fromtxg, TRAVERSE_PRE | TRAVERSE_PREFETCH,
 470             backup_cb, &ba);
 471
 472         if (ba.pending_op != PENDING_NONE)
 473                 if (dump_bytes(&ba, drr, sizeof (dmu_replay_record_t)) != 0)
 474                         err = EINTR;
 475
 476         if (err) {
 477                 if (err == EINTR && ba.err)
 478                         err = ba.err;
 479                 kmem_free(drr, sizeof (dmu_replay_record_t));
 480                 return (err);
 481         }
 482
 483         bzero(drr, sizeof (dmu_replay_record_t));
 484         drr->drr_type = DRR_END;
 485         drr->drr_u.drr_end.drr_checksum = ba.zc;
 486         drr->drr_u.drr_end.drr_toguid = ba.toguid;
 487
 488         if (dump_bytes(&ba, drr, sizeof (dmu_replay_record_t)) != 0) {
 489                 kmem_free(drr, sizeof (dmu_replay_record_t));
 490                 return (ba.err);
 491         }
 492
 493         kmem_free(drr, sizeof (dmu_replay_record_t));
 494
 495         return (0);
 496 }
 497
 498 struct recvbeginsyncarg {
 499         const char *tofs;
 500         const char *tosnap;
 501         dsl_dataset_t *origin;
 502         uint64_t fromguid;
 503         dmu_objset_type_t type;
 504         void *tag;
 505         boolean_t force;
 506         uint64_t dsflags;
 507         char clonelastname[MAXNAMELEN];
 508         dsl_dataset_t *ds; /* the ds to recv into; returned from the syncfunc */
 509         cred_t *cr;
 510 };
 511
 512 /* ARGSUSED */
 513 static int
 514 recv_new_check(void *arg1, void *arg2, dmu_tx_t *tx)
 515 {
 516         dsl_dir_t *dd = arg1;
 517         struct recvbeginsyncarg *rbsa = arg2;
 518         objset_t *mos = dd->dd_pool->dp_meta_objset;
 519         uint64_t val;
 520         int err;
 521
 522         err = zap_lookup(mos, dd->dd_phys->dd_child_dir_zapobj,
 523             strrchr(rbsa->tofs, '/') + 1, sizeof (uint64_t), 1, &val);
 524
 525         if (err != ENOENT)
 526                 return (err ? err : EEXIST);
 527
 528         if (rbsa->origin) {
 529                 /* make sure it's a snap in the same pool */
 530                 if (rbsa->origin->ds_dir->dd_pool != dd->dd_pool)
 531                         return (EXDEV);
 532                 if (!dsl_dataset_is_snapshot(rbsa->origin))
 533                         return (EINVAL);
 534                 if (rbsa->origin->ds_phys->ds_guid != rbsa->fromguid)
 535                         return (ENODEV);
 536         }
 537
 538         return (0);
 539 }
 540
 541 static void
 542 recv_new_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 543 {
 544         dsl_dir_t *dd = arg1;
 545         struct recvbeginsyncarg *rbsa = arg2;
 546         uint64_t flags = DS_FLAG_INCONSISTENT | rbsa->dsflags;
 547         uint64_t dsobj;
 548
 549         /* Create and open new dataset. */
 550         dsobj = dsl_dataset_create_sync(dd, strrchr(rbsa->tofs, '/') + 1,
 551             rbsa->origin, flags, rbsa->cr, tx);
 552         VERIFY(0 == dsl_dataset_own_obj(dd->dd_pool, dsobj,
 553             B_TRUE, dmu_recv_tag, &rbsa->ds));
 554
 555         if (rbsa->origin == NULL) {
 556                 (void) dmu_objset_create_impl(dd->dd_pool->dp_spa,
 557                     rbsa->ds, &rbsa->ds->ds_phys->ds_bp, rbsa->type, tx);
 558         }
 559
 560         spa_history_log_internal(LOG_DS_REPLAY_FULL_SYNC,
 561             dd->dd_pool->dp_spa, tx, "dataset = %lld", dsobj);
 562 }
 563
 564 /* ARGSUSED */
 565 static int
 566 recv_existing_check(void *arg1, void *arg2, dmu_tx_t *tx)
 567 {
 568         dsl_dataset_t *ds = arg1;
 569         struct recvbeginsyncarg *rbsa = arg2;
 570         int err;
 571         uint64_t val;
 572
 573         /* must not have any changes since most recent snapshot */
 574         if (!rbsa->force && dsl_dataset_modified_since_lastsnap(ds))
 575                 return (ETXTBSY);
 576
 577         /* new snapshot name must not exist */
 578         err = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset,
 579             ds->ds_phys->ds_snapnames_zapobj, rbsa->tosnap, 8, 1, &val);
 580         if (err == 0)
 581                 return (EEXIST);
 582         if (err != ENOENT)
 583                 return (err);
 584
 585         if (rbsa->fromguid) {
 586                 /* if incremental, most recent snapshot must match fromguid */
 587                 if (ds->ds_prev == NULL)
 588                         return (ENODEV);
 589
 590                 /*
 591                  * most recent snapshot must match fromguid, or there are no
 592                  * changes since the fromguid one
 593                  */
 594                 if (ds->ds_prev->ds_phys->ds_guid != rbsa->fromguid) {
 595                         uint64_t birth = ds->ds_prev->ds_phys->ds_bp.blk_birth;
 596                         uint64_t obj = ds->ds_prev->ds_phys->ds_prev_snap_obj;
 597                         while (obj != 0) {
 598                                 dsl_dataset_t *snap;
 599                                 err = dsl_dataset_hold_obj(ds->ds_dir->dd_pool,
 600                                     obj, FTAG, &snap);
 601                                 if (err)
 602                                         return (ENODEV);
 603                                 if (snap->ds_phys->ds_creation_txg < birth) {
 604                                         dsl_dataset_rele(snap, FTAG);
 605                                         return (ENODEV);
 606                                 }
 607                                 if (snap->ds_phys->ds_guid == rbsa->fromguid) {
 608                                         dsl_dataset_rele(snap, FTAG);
 609                                         break; /* it's ok */
 610                                 }
 611                                 obj = snap->ds_phys->ds_prev_snap_obj;
 612                                 dsl_dataset_rele(snap, FTAG);
 613                         }
 614                         if (obj == 0)
 615                                 return (ENODEV);
 616                 }
 617         } else {
 618                 /* if full, most recent snapshot must be $ORIGIN */
 619                 if (ds->ds_phys->ds_prev_snap_txg >= TXG_INITIAL)
 620                         return (ENODEV);
 621         }
 622
 623         /* temporary clone name must not exist */
 624         err = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset,
 625             ds->ds_dir->dd_phys->dd_child_dir_zapobj,
 626             rbsa->clonelastname, 8, 1, &val);
 627         if (err == 0)
 628                 return (EEXIST);
 629         if (err != ENOENT)
 630                 return (err);
 631
 632         return (0);
 633 }
 634
 635 /* ARGSUSED */
 636 static void
 637 recv_existing_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 638 {
 639         dsl_dataset_t *ohds = arg1;
 640         struct recvbeginsyncarg *rbsa = arg2;
 641         dsl_pool_t *dp = ohds->ds_dir->dd_pool;
 642         dsl_dataset_t *cds;
 643         uint64_t flags = DS_FLAG_INCONSISTENT | rbsa->dsflags;
 644         uint64_t dsobj;
 645
 646         /* create and open the temporary clone */
 647         dsobj = dsl_dataset_create_sync(ohds->ds_dir, rbsa->clonelastname,
 648             ohds->ds_prev, flags, rbsa->cr, tx);
 649         VERIFY(0 == dsl_dataset_own_obj(dp, dsobj, B_TRUE, dmu_recv_tag, &cds));
 650
 651         /*
 652          * If we actually created a non-clone, we need to create the
 653          * objset in our new dataset.
 654          */
 655         if (BP_IS_HOLE(dsl_dataset_get_blkptr(cds))) {
 656                 (void) dmu_objset_create_impl(dp->dp_spa,
 657                     cds, dsl_dataset_get_blkptr(cds), rbsa->type, tx);
 658         }
 659
 660         rbsa->ds = cds;
 661
 662         spa_history_log_internal(LOG_DS_REPLAY_INC_SYNC,
 663             dp->dp_spa, tx, "dataset = %lld", dsobj);
 664 }
 665
 666 static boolean_t
 667 dmu_recv_verify_features(dsl_dataset_t *ds, struct drr_begin *drrb)
 668 {
 669         int featureflags;
 670
 671         featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
 672
 673         /* Verify pool version supports SA if SA_SPILL feature set */
 674         return ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
 675             (spa_version(dsl_dataset_get_spa(ds)) < SPA_VERSION_SA));
 676 }
 677
 678 /*
 679  * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin()
 680  * succeeds; otherwise we will leak the holds on the datasets.
 681  */
 682 int
 683 dmu_recv_begin(char *tofs, char *tosnap, char *top_ds, struct drr_begin *drrb,
 684     boolean_t force, objset_t *origin, dmu_recv_cookie_t *drc)
 685 {
 686         int err = 0;
 687         boolean_t byteswap;
 688         struct recvbeginsyncarg rbsa = { 0 };
 689         uint64_t versioninfo;
 690         int flags;
 691         dsl_dataset_t *ds;
 692
 693         if (drrb->drr_magic == DMU_BACKUP_MAGIC)
 694                 byteswap = FALSE;
 695         else if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC))
 696                 byteswap = TRUE;
 697         else
 698                 return (EINVAL);
 699
 700         rbsa.tofs = tofs;
 701         rbsa.tosnap = tosnap;
 702         rbsa.origin = origin ? origin->os_dsl_dataset : NULL;
 703         rbsa.fromguid = drrb->drr_fromguid;
 704         rbsa.type = drrb->drr_type;
 705         rbsa.tag = FTAG;
 706         rbsa.dsflags = 0;
 707         rbsa.cr = CRED();
 708         versioninfo = drrb->drr_versioninfo;
 709         flags = drrb->drr_flags;
 710
 711         if (byteswap) {
 712                 rbsa.type = BSWAP_32(rbsa.type);
 713                 rbsa.fromguid = BSWAP_64(rbsa.fromguid);
 714                 versioninfo = BSWAP_64(versioninfo);
 715                 flags = BSWAP_32(flags);
 716         }
 717
 718         if (DMU_GET_STREAM_HDRTYPE(versioninfo) == DMU_COMPOUNDSTREAM ||
 719             rbsa.type >= DMU_OST_NUMTYPES ||
 720             ((flags & DRR_FLAG_CLONE) && origin == NULL))
 721                 return (EINVAL);
 722
 723         if (flags & DRR_FLAG_CI_DATA)
 724                 rbsa.dsflags = DS_FLAG_CI_DATASET;
 725
 726         bzero(drc, sizeof (dmu_recv_cookie_t));
 727         drc->drc_drrb = drrb;
 728         drc->drc_tosnap = tosnap;
 729         drc->drc_top_ds = top_ds;
 730         drc->drc_force = force;
 731
 732         /*
 733          * Process the begin in syncing context.
 734          */
 735
 736         /* open the dataset we are logically receiving into */
 737         err = dsl_dataset_hold(tofs, dmu_recv_tag, &ds);
 738         if (err == 0) {
 739                 if (dmu_recv_verify_features(ds, drrb)) {
 740                         dsl_dataset_rele(ds, dmu_recv_tag);
 741                         return (ENOTSUP);
 742                 }
 743                 /* target fs already exists; recv into temp clone */
 744
 745                 /* Can't recv a clone into an existing fs */
 746                 if (flags & DRR_FLAG_CLONE) {
 747                         dsl_dataset_rele(ds, dmu_recv_tag);
 748                         return (EINVAL);
 749                 }
 750
 751                 /* must not have an incremental recv already in progress */
 752                 if (!mutex_tryenter(&ds->ds_recvlock)) {
 753                         dsl_dataset_rele(ds, dmu_recv_tag);
 754                         return (EBUSY);
 755                 }
 756
 757                 /* tmp clone name is: tofs/%tosnap" */
 758                 (void) snprintf(rbsa.clonelastname, sizeof (rbsa.clonelastname),
 759                     "%%%s", tosnap);
 760                 rbsa.force = force;
 761                 err = dsl_sync_task_do(ds->ds_dir->dd_pool,
 762                     recv_existing_check, recv_existing_sync, ds, &rbsa, 5);
 763                 if (err) {
 764                         mutex_exit(&ds->ds_recvlock);
 765                         dsl_dataset_rele(ds, dmu_recv_tag);
 766                         return (err);
 767                 }
 768                 drc->drc_logical_ds = ds;
 769                 drc->drc_real_ds = rbsa.ds;
 770         } else if (err == ENOENT) {
 771                 /* target fs does not exist; must be a full backup or clone */
 772                 char *cp;
 773
 774                 /*
 775                  * If it's a non-clone incremental, we are missing the
 776                  * target fs, so fail the recv.
 777                  */
 778                 if (rbsa.fromguid && !(flags & DRR_FLAG_CLONE))
 779                         return (ENOENT);
 780
 781                 /* Open the parent of tofs */
 782                 cp = strrchr(tofs, '/');
 783                 *cp = '\0';
 784                 err = dsl_dataset_hold(tofs, FTAG, &ds);
 785                 *cp = '/';
 786                 if (err)
 787                         return (err);
 788
 789                 if (dmu_recv_verify_features(ds, drrb)) {
 790                         dsl_dataset_rele(ds, FTAG);
 791                         return (ENOTSUP);
 792                 }
 793
 794                 err = dsl_sync_task_do(ds->ds_dir->dd_pool,
 795                     recv_new_check, recv_new_sync, ds->ds_dir, &rbsa, 5);
 796                 dsl_dataset_rele(ds, FTAG);
 797                 if (err)
 798                         return (err);
 799                 drc->drc_logical_ds = drc->drc_real_ds = rbsa.ds;
 800                 drc->drc_newfs = B_TRUE;
 801         }
 802
 803         return (err);
 804 }
 805
 806 struct restorearg {
 807         int err;
 808         int byteswap;
 809         vnode_t *vp;
 810         char *buf;
 811         uint64_t voff;
 812         int bufsize; /* amount of memory allocated for buf */
 813         zio_cksum_t cksum;
 814         avl_tree_t *guid_to_ds_map;
 815 };
 816
 817 typedef struct guid_map_entry {
 818         uint64_t        guid;
 819         dsl_dataset_t   *gme_ds;
 820         avl_node_t      avlnode;
 821 } guid_map_entry_t;
 822
 823 static int
 824 guid_compare(const void *arg1, const void *arg2)
 825 {
 826         const guid_map_entry_t *gmep1 = arg1;
 827         const guid_map_entry_t *gmep2 = arg2;
 828
 829         if (gmep1->guid < gmep2->guid)
 830                 return (-1);
 831         else if (gmep1->guid > gmep2->guid)
 832                 return (1);
 833         return (0);
 834 }
 835
 836 /*
 837  * This function is a callback used by dmu_objset_find() (which
 838  * enumerates the object sets) to build an avl tree that maps guids
 839  * to datasets.  The resulting table is used when processing DRR_WRITE_BYREF
 840  * send stream records.  These records, which are used in dedup'ed
 841  * streams, do not contain data themselves, but refer to a copy
 842  * of the data block that has already been written because it was
 843  * earlier in the stream.  That previous copy is identified by the
 844  * guid of the dataset with the referenced data.
 845  */
 846 int
 847 find_ds_by_guid(const char *name, void *arg)
 848 {
 849         avl_tree_t *guid_map = arg;
 850         dsl_dataset_t *ds, *snapds;
 851         guid_map_entry_t *gmep;
 852         dsl_pool_t *dp;
 853         int err;
 854         uint64_t lastobj, firstobj;
 855
 856         if (dsl_dataset_hold(name, FTAG, &ds) != 0)
 857                 return (0);
 858
 859         dp = ds->ds_dir->dd_pool;
 860         rw_enter(&dp->dp_config_rwlock, RW_READER);
 861         firstobj = ds->ds_dir->dd_phys->dd_origin_obj;
 862         lastobj = ds->ds_phys->ds_prev_snap_obj;
 863
 864         while (lastobj != firstobj) {
 865                 err = dsl_dataset_hold_obj(dp, lastobj, guid_map, &snapds);
 866                 if (err) {
 867                         /*
 868                          * Skip this snapshot and move on. It's not
 869                          * clear why this would ever happen, but the
 870                          * remainder of the snapshot streadm can be
 871                          * processed.
 872                          */
 873                         rw_exit(&dp->dp_config_rwlock);
 874                         dsl_dataset_rele(ds, FTAG);
 875                         return (0);
 876                 }
 877
 878                 gmep = kmem_alloc(sizeof (guid_map_entry_t), KM_SLEEP);
 879                 gmep->guid = snapds->ds_phys->ds_guid;
 880                 gmep->gme_ds = snapds;
 881                 avl_add(guid_map, gmep);
 882                 lastobj = snapds->ds_phys->ds_prev_snap_obj;
 883         }
 884
 885         rw_exit(&dp->dp_config_rwlock);
 886         dsl_dataset_rele(ds, FTAG);
 887
 888         return (0);
 889 }
 890
 891 static void
 892 free_guid_map_onexit(void *arg)
 893 {
 894         avl_tree_t *ca = arg;
 895         void *cookie = NULL;
 896         guid_map_entry_t *gmep;
 897
 898         while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) {
 899                 dsl_dataset_rele(gmep->gme_ds, ca);
 900                 kmem_free(gmep, sizeof (guid_map_entry_t));
 901         }
 902         avl_destroy(ca);
 903         kmem_free(ca, sizeof (avl_tree_t));
 904 }
 905
 906 static void *
 907 restore_read(struct restorearg *ra, int len)
 908 {
 909         void *rv;
 910         int done = 0;
 911
 912         /* some things will require 8-byte alignment, so everything must */
 913         ASSERT3U(len % 8, ==, 0);
 914
 915         while (done < len) {
 916                 ssize_t resid;
 917
 918                 ra->err = vn_rdwr(UIO_READ, ra->vp,
 919                     (caddr_t)ra->buf + done, len - done,
 920                     ra->voff, UIO_SYSSPACE, FAPPEND,
 921                     RLIM64_INFINITY, CRED(), &resid);
 922
 923                 if (resid == len - done)
 924                         ra->err = EINVAL;
 925                 ra->voff += len - done - resid;
 926                 done = len - resid;
 927                 if (ra->err)
 928                         return (NULL);
 929         }
 930
 931         ASSERT3U(done, ==, len);
 932         rv = ra->buf;
 933         if (ra->byteswap)
 934                 fletcher_4_incremental_byteswap(rv, len, &ra->cksum);
 935         else
 936                 fletcher_4_incremental_native(rv, len, &ra->cksum);
 937         return (rv);
 938 }
 939
 940 static void
 941 backup_byteswap(dmu_replay_record_t *drr)
 942 {
 943 #define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
 944 #define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
 945         drr->drr_type = BSWAP_32(drr->drr_type);
 946         drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);
 947         switch (drr->drr_type) {
 948         case DRR_BEGIN:
 949                 DO64(drr_begin.drr_magic);
 950                 DO64(drr_begin.drr_versioninfo);
 951                 DO64(drr_begin.drr_creation_time);
 952                 DO32(drr_begin.drr_type);
 953                 DO32(drr_begin.drr_flags);
 954                 DO64(drr_begin.drr_toguid);
 955                 DO64(drr_begin.drr_fromguid);
 956                 break;
 957         case DRR_OBJECT:
 958                 DO64(drr_object.drr_object);
 959                 /* DO64(drr_object.drr_allocation_txg); */
 960                 DO32(drr_object.drr_type);
 961                 DO32(drr_object.drr_bonustype);
 962                 DO32(drr_object.drr_blksz);
 963                 DO32(drr_object.drr_bonuslen);
 964                 DO64(drr_object.drr_toguid);
 965                 break;
 966         case DRR_FREEOBJECTS:
 967                 DO64(drr_freeobjects.drr_firstobj);
 968                 DO64(drr_freeobjects.drr_numobjs);
 969                 DO64(drr_freeobjects.drr_toguid);
 970                 break;
 971         case DRR_WRITE:
 972                 DO64(drr_write.drr_object);
 973                 DO32(drr_write.drr_type);
 974                 DO64(drr_write.drr_offset);
 975                 DO64(drr_write.drr_length);
 976                 DO64(drr_write.drr_toguid);
 977                 DO64(drr_write.drr_key.ddk_cksum.zc_word[0]);
 978                 DO64(drr_write.drr_key.ddk_cksum.zc_word[1]);
 979                 DO64(drr_write.drr_key.ddk_cksum.zc_word[2]);
 980                 DO64(drr_write.drr_key.ddk_cksum.zc_word[3]);
 981                 DO64(drr_write.drr_key.ddk_prop);
 982                 break;
 983         case DRR_WRITE_BYREF:
 984                 DO64(drr_write_byref.drr_object);
 985                 DO64(drr_write_byref.drr_offset);
 986                 DO64(drr_write_byref.drr_length);
 987                 DO64(drr_write_byref.drr_toguid);
 988                 DO64(drr_write_byref.drr_refguid);
 989                 DO64(drr_write_byref.drr_refobject);
 990                 DO64(drr_write_byref.drr_refoffset);
 991                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[0]);
 992                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[1]);
 993                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[2]);
 994                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[3]);
 995                 DO64(drr_write_byref.drr_key.ddk_prop);
 996                 break;
 997         case DRR_FREE:
 998                 DO64(drr_free.drr_object);
 999                 DO64(drr_free.drr_offset);
1000                 DO64(drr_free.drr_length);
1001                 DO64(drr_free.drr_toguid);
1002                 break;
1003         case DRR_SPILL:
1004                 DO64(drr_spill.drr_object);
1005                 DO64(drr_spill.drr_length);
1006                 DO64(drr_spill.drr_toguid);
1007                 break;
1008         case DRR_END:
1009                 DO64(drr_end.drr_checksum.zc_word[0]);
1010                 DO64(drr_end.drr_checksum.zc_word[1]);
1011                 DO64(drr_end.drr_checksum.zc_word[2]);
1012                 DO64(drr_end.drr_checksum.zc_word[3]);
1013                 DO64(drr_end.drr_toguid);
1014                 break;
1015         default:
1016                 break;
1017         }
1018 #undef DO64
1019 #undef DO32
1020 }
1021
1022 static int
1023 restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
1024 {
1025         int err;
1026         dmu_tx_t *tx;
1027         void *data = NULL;
1028
1029         if (drro->drr_type == DMU_OT_NONE ||
1030             drro->drr_type >= DMU_OT_NUMTYPES ||
1031             drro->drr_bonustype >= DMU_OT_NUMTYPES ||
1032             drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS ||
1033             drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
1034             P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
1035             drro->drr_blksz < SPA_MINBLOCKSIZE ||
1036             drro->drr_blksz > SPA_MAXBLOCKSIZE ||
1037             drro->drr_bonuslen > DN_MAX_BONUSLEN) {
1038                 return (EINVAL);
1039         }
1040
1041         err = dmu_object_info(os, drro->drr_object, NULL);
1042
1043         if (err != 0 && err != ENOENT)
1044                 return (EINVAL);
1045
1046         if (drro->drr_bonuslen) {
1047                 data = restore_read(ra, P2ROUNDUP(drro->drr_bonuslen, 8));
1048                 if (ra->err)
1049                         return (ra->err);
1050         }
1051
1052         if (err == ENOENT) {
1053                 /* currently free, want to be allocated */
1054                 tx = dmu_tx_create(os);
1055                 dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1056                 err = dmu_tx_assign(tx, TXG_WAIT);
1057                 if (err) {
1058                         dmu_tx_abort(tx);
1059                         return (err);
1060                 }
1061                 err = dmu_object_claim(os, drro->drr_object,
1062                     drro->drr_type, drro->drr_blksz,
1063                     drro->drr_bonustype, drro->drr_bonuslen, tx);
1064                 dmu_tx_commit(tx);
1065         } else {
1066                 /* currently allocated, want to be allocated */
1067                 err = dmu_object_reclaim(os, drro->drr_object,
1068                     drro->drr_type, drro->drr_blksz,
1069                     drro->drr_bonustype, drro->drr_bonuslen);
1070         }
1071         if (err) {
1072                 return (EINVAL);
1073         }
1074
1075         tx = dmu_tx_create(os);
1076         dmu_tx_hold_bonus(tx, drro->drr_object);
1077         err = dmu_tx_assign(tx, TXG_WAIT);
1078         if (err) {
1079                 dmu_tx_abort(tx);
1080                 return (err);
1081         }
1082
1083         dmu_object_set_checksum(os, drro->drr_object, drro->drr_checksumtype,
1084             tx);
1085         dmu_object_set_compress(os, drro->drr_object, drro->drr_compress, tx);
1086
1087         if (data != NULL) {
1088                 dmu_buf_t *db;
1089
1090                 VERIFY(0 == dmu_bonus_hold(os, drro->drr_object, FTAG, &db));
1091                 dmu_buf_will_dirty(db, tx);
1092
1093                 ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
1094                 bcopy(data, db->db_data, drro->drr_bonuslen);
1095                 if (ra->byteswap) {
1096                         dmu_ot[drro->drr_bonustype].ot_byteswap(db->db_data,
1097                             drro->drr_bonuslen);
1098                 }
1099                 dmu_buf_rele(db, FTAG);
1100         }
1101         dmu_tx_commit(tx);
1102         return (0);
1103 }
1104
1105 /* ARGSUSED */
1106 static int
1107 restore_freeobjects(struct restorearg *ra, objset_t *os,
1108     struct drr_freeobjects *drrfo)
1109 {
1110         uint64_t obj;
1111
1112         if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj)
1113                 return (EINVAL);
1114
1115         for (obj = drrfo->drr_firstobj;
1116             obj < drrfo->drr_firstobj + drrfo->drr_numobjs;
1117             (void) dmu_object_next(os, &obj, FALSE, 0)) {
1118                 int err;
1119
1120                 if (dmu_object_info(os, obj, NULL) != 0)
1121                         continue;
1122
1123                 err = dmu_free_object(os, obj);
1124                 if (err)
1125                         return (err);
1126         }
1127         return (0);
1128 }
1129
1130 static int
1131 restore_write(struct restorearg *ra, objset_t *os,
1132     struct drr_write *drrw)
1133 {
1134         dmu_tx_t *tx;
1135         void *data;
1136         int err;
1137
1138         if (drrw->drr_offset + drrw->drr_length < drrw->drr_offset ||
1139             drrw->drr_type >= DMU_OT_NUMTYPES)
1140                 return (EINVAL);
1141
1142         data = restore_read(ra, drrw->drr_length);
1143         if (data == NULL)
1144                 return (ra->err);
1145
1146         if (dmu_object_info(os, drrw->drr_object, NULL) != 0)
1147                 return (EINVAL);
1148
1149         tx = dmu_tx_create(os);
1150
1151         dmu_tx_hold_write(tx, drrw->drr_object,
1152             drrw->drr_offset, drrw->drr_length);
1153         err = dmu_tx_assign(tx, TXG_WAIT);
1154         if (err) {
1155                 dmu_tx_abort(tx);
1156                 return (err);
1157         }
1158         if (ra->byteswap)
1159                 dmu_ot[drrw->drr_type].ot_byteswap(data, drrw->drr_length);
1160         dmu_write(os, drrw->drr_object,
1161             drrw->drr_offset, drrw->drr_length, data, tx);
1162         dmu_tx_commit(tx);
1163         return (0);
1164 }
1165
1166 /*
1167  * Handle a DRR_WRITE_BYREF record.  This record is used in dedup'ed
1168  * streams to refer to a copy of the data that is already on the
1169  * system because it came in earlier in the stream.  This function
1170  * finds the earlier copy of the data, and uses that copy instead of
1171  * data from the stream to fulfill this write.
1172  */
1173 static int
1174 restore_write_byref(struct restorearg *ra, objset_t *os,
1175     struct drr_write_byref *drrwbr)
1176 {
1177         dmu_tx_t *tx;
1178         int err;
1179         guid_map_entry_t gmesrch;
1180         guid_map_entry_t *gmep;
1181         avl_index_t     where;
1182         objset_t *ref_os = NULL;
1183         dmu_buf_t *dbp;
1184
1185         if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset)
1186                 return (EINVAL);
1187
1188         /*
1189          * If the GUID of the referenced dataset is different from the
1190          * GUID of the target dataset, find the referenced dataset.
1191          */
1192         if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
1193                 gmesrch.guid = drrwbr->drr_refguid;
1194                 if ((gmep = avl_find(ra->guid_to_ds_map, &gmesrch,
1195                     &where)) == NULL) {
1196                         return (EINVAL);
1197                 }
1198                 if (dmu_objset_from_ds(gmep->gme_ds, &ref_os))
1199                         return (EINVAL);
1200         } else {
1201                 ref_os = os;
1202         }
1203
1204         if (err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
1205             drrwbr->drr_refoffset, FTAG, &dbp, DMU_READ_PREFETCH))
1206                 return (err);
1207
1208         tx = dmu_tx_create(os);
1209
1210         dmu_tx_hold_write(tx, drrwbr->drr_object,
1211             drrwbr->drr_offset, drrwbr->drr_length);
1212         err = dmu_tx_assign(tx, TXG_WAIT);
1213         if (err) {
1214                 dmu_tx_abort(tx);
1215                 return (err);
1216         }
1217         dmu_write(os, drrwbr->drr_object,
1218             drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
1219         dmu_buf_rele(dbp, FTAG);
1220         dmu_tx_commit(tx);
1221         return (0);
1222 }
1223
1224 static int
1225 restore_spill(struct restorearg *ra, objset_t *os, struct drr_spill *drrs)
1226 {
1227         dmu_tx_t *tx;
1228         void *data;
1229         dmu_buf_t *db, *db_spill;
1230         int err;
1231
1232         if (drrs->drr_length < SPA_MINBLOCKSIZE ||
1233             drrs->drr_length > SPA_MAXBLOCKSIZE)
1234                 return (EINVAL);
1235
1236         data = restore_read(ra, drrs->drr_length);
1237         if (data == NULL)
1238                 return (ra->err);
1239
1240         if (dmu_object_info(os, drrs->drr_object, NULL) != 0)
1241                 return (EINVAL);
1242
1243         VERIFY(0 == dmu_bonus_hold(os, drrs->drr_object, FTAG, &db));
1244         if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
1245                 dmu_buf_rele(db, FTAG);
1246                 return (err);
1247         }
1248
1249         tx = dmu_tx_create(os);
1250
1251         dmu_tx_hold_spill(tx, db->db_object);
1252
1253         err = dmu_tx_assign(tx, TXG_WAIT);
1254         if (err) {
1255                 dmu_buf_rele(db, FTAG);
1256                 dmu_buf_rele(db_spill, FTAG);
1257                 dmu_tx_abort(tx);
1258                 return (err);
1259         }
1260         dmu_buf_will_dirty(db_spill, tx);
1261
1262         if (db_spill->db_size < drrs->drr_length)
1263                 VERIFY(0 == dbuf_spill_set_blksz(db_spill,
1264                     drrs->drr_length, tx));
1265         bcopy(data, db_spill->db_data, drrs->drr_length);
1266
1267         dmu_buf_rele(db, FTAG);
1268         dmu_buf_rele(db_spill, FTAG);
1269
1270         dmu_tx_commit(tx);
1271         return (0);
1272 }
1273
1274 /* ARGSUSED */
1275 static int
1276 restore_free(struct restorearg *ra, objset_t *os,
1277     struct drr_free *drrf)
1278 {
1279         int err;
1280
1281         if (drrf->drr_length != -1ULL &&
1282             drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
1283                 return (EINVAL);
1284
1285         if (dmu_object_info(os, drrf->drr_object, NULL) != 0)
1286                 return (EINVAL);
1287
1288         err = dmu_free_long_range(os, drrf->drr_object,
1289             drrf->drr_offset, drrf->drr_length);
1290         return (err);
1291 }
1292
1293 /*
1294  * NB: callers *must* call dmu_recv_end() if this succeeds.
1295  */
1296 int
1297 dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
1298     int cleanup_fd, uint64_t *action_handlep)
1299 {
1300         struct restorearg ra = { 0 };
1301         dmu_replay_record_t *drr;
1302         objset_t *os;
1303         zio_cksum_t pcksum;
1304         int featureflags;
1305
1306         if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC))
1307                 ra.byteswap = TRUE;
1308
1309         {
1310                 /* compute checksum of drr_begin record */
1311                 dmu_replay_record_t *drr;
1312                 drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
1313
1314                 drr->drr_type = DRR_BEGIN;
1315                 drr->drr_u.drr_begin = *drc->drc_drrb;
1316                 if (ra.byteswap) {
1317                         fletcher_4_incremental_byteswap(drr,
1318                             sizeof (dmu_replay_record_t), &ra.cksum);
1319                 } else {
1320                         fletcher_4_incremental_native(drr,
1321                             sizeof (dmu_replay_record_t), &ra.cksum);
1322                 }
1323                 kmem_free(drr, sizeof (dmu_replay_record_t));
1324         }
1325
1326         if (ra.byteswap) {
1327                 struct drr_begin *drrb = drc->drc_drrb;
1328                 drrb->drr_magic = BSWAP_64(drrb->drr_magic);
1329                 drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
1330                 drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
1331                 drrb->drr_type = BSWAP_32(drrb->drr_type);
1332                 drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
1333                 drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
1334         }
1335
1336         ra.vp = vp;
1337         ra.voff = *voffp;
1338         ra.bufsize = 1<<20;
1339         ra.buf = kmem_alloc(ra.bufsize, KM_SLEEP);
1340
1341         /* these were verified in dmu_recv_begin */
1342         ASSERT(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo) ==
1343             DMU_SUBSTREAM);
1344         ASSERT(drc->drc_drrb->drr_type < DMU_OST_NUMTYPES);
1345
1346         /*
1347          * Open the objset we are modifying.
1348          */
1349         VERIFY(dmu_objset_from_ds(drc->drc_real_ds, &os) == 0);
1350
1351         ASSERT(drc->drc_real_ds->ds_phys->ds_flags & DS_FLAG_INCONSISTENT);
1352
1353         featureflags = DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);
1354
1355         /* if this stream is dedup'ed, set up the avl tree for guid mapping */
1356         if (featureflags & DMU_BACKUP_FEATURE_DEDUP) {
1357                 minor_t minor;
1358
1359                 if (cleanup_fd == -1) {
1360                         ra.err = EBADF;
1361                         goto out;
1362                 }
1363                 ra.err = zfs_onexit_fd_hold(cleanup_fd, &minor);
1364                 if (ra.err) {
1365                         cleanup_fd = -1;
1366                         goto out;
1367                 }
1368
1369                 if (*action_handlep == 0) {
1370                         ra.guid_to_ds_map =
1371                             kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
1372                         avl_create(ra.guid_to_ds_map, guid_compare,
1373                             sizeof (guid_map_entry_t),
1374                             offsetof(guid_map_entry_t, avlnode));
1375                         (void) dmu_objset_find(drc->drc_top_ds, find_ds_by_guid,
1376                             (void *)ra.guid_to_ds_map,
1377                             DS_FIND_CHILDREN);
1378                         ra.err = zfs_onexit_add_cb(minor,
1379                             free_guid_map_onexit, ra.guid_to_ds_map,
1380                             action_handlep);
1381                         if (ra.err)
1382                                 goto out;
1383                 } else {
1384                         ra.err = zfs_onexit_cb_data(minor, *action_handlep,
1385                             (void **)&ra.guid_to_ds_map);
1386                         if (ra.err)
1387                                 goto out;
1388                 }
1389         }
1390
1391         /*
1392          * Read records and process them.
1393          */
1394         pcksum = ra.cksum;
1395         while (ra.err == 0 &&
1396             NULL != (drr = restore_read(&ra, sizeof (*drr)))) {
1397                 if (issig(JUSTLOOKING) && issig(FORREAL)) {
1398                         ra.err = EINTR;
1399                         goto out;
1400                 }
1401
1402                 if (ra.byteswap)
1403                         backup_byteswap(drr);
1404
1405                 switch (drr->drr_type) {
1406                 case DRR_OBJECT:
1407                 {
1408                         /*
1409                          * We need to make a copy of the record header,
1410                          * because restore_{object,write} may need to
1411                          * restore_read(), which will invalidate drr.
1412                          */
1413                         struct drr_object drro = drr->drr_u.drr_object;
1414                         ra.err = restore_object(&ra, os, &drro);
1415                         break;
1416                 }
1417                 case DRR_FREEOBJECTS:
1418                 {
1419                         struct drr_freeobjects drrfo =
1420                             drr->drr_u.drr_freeobjects;
1421                         ra.err = restore_freeobjects(&ra, os, &drrfo);
1422                         break;
1423                 }
1424                 case DRR_WRITE:
1425                 {
1426                         struct drr_write drrw = drr->drr_u.drr_write;
1427                         ra.err = restore_write(&ra, os, &drrw);
1428                         break;
1429                 }
1430                 case DRR_WRITE_BYREF:
1431                 {
1432                         struct drr_write_byref drrwbr =
1433                             drr->drr_u.drr_write_byref;
1434                         ra.err = restore_write_byref(&ra, os, &drrwbr);
1435                         break;
1436                 }
1437                 case DRR_FREE:
1438                 {
1439                         struct drr_free drrf = drr->drr_u.drr_free;
1440                         ra.err = restore_free(&ra, os, &drrf);
1441                         break;
1442                 }
1443                 case DRR_END:
1444                 {
1445                         struct drr_end drre = drr->drr_u.drr_end;
1446                         /*
1447                          * We compare against the *previous* checksum
1448                          * value, because the stored checksum is of
1449                          * everything before the DRR_END record.
1450                          */
1451                         if (!ZIO_CHECKSUM_EQUAL(drre.drr_checksum, pcksum))
1452                                 ra.err = ECKSUM;
1453                         goto out;
1454                 }
1455                 case DRR_SPILL:
1456                 {
1457                         struct drr_spill drrs = drr->drr_u.drr_spill;
1458                         ra.err = restore_spill(&ra, os, &drrs);
1459                         break;
1460                 }
1461                 default:
1462                         ra.err = EINVAL;
1463                         goto out;
1464                 }
1465                 pcksum = ra.cksum;
1466         }
1467         ASSERT(ra.err != 0);
1468
1469 out:
1470         if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
1471                 zfs_onexit_fd_rele(cleanup_fd);
1472
1473         if (ra.err != 0) {
1474                 /*
1475                  * destroy what we created, so we don't leave it in the
1476                  * inconsistent restoring state.
1477                  */
1478                 txg_wait_synced(drc->drc_real_ds->ds_dir->dd_pool, 0);
1479
1480                 (void) dsl_dataset_destroy(drc->drc_real_ds, dmu_recv_tag,
1481                     B_FALSE);
1482                 if (drc->drc_real_ds != drc->drc_logical_ds) {
1483                         mutex_exit(&drc->drc_logical_ds->ds_recvlock);
1484                         dsl_dataset_rele(drc->drc_logical_ds, dmu_recv_tag);
1485                 }
1486         }
1487
1488         kmem_free(ra.buf, ra.bufsize);
1489         *voffp = ra.voff;
1490         return (ra.err);
1491 }
1492
1493 struct recvendsyncarg {
1494         char *tosnap;
1495         uint64_t creation_time;
1496         uint64_t toguid;
1497 };
1498
1499 static int
1500 recv_end_check(void *arg1, void *arg2, dmu_tx_t *tx)
1501 {
1502         dsl_dataset_t *ds = arg1;
1503         struct recvendsyncarg *resa = arg2;
1504
1505         return (dsl_dataset_snapshot_check(ds, resa->tosnap, tx));
1506 }
1507
1508 static void
1509 recv_end_sync(void *arg1, void *arg2, dmu_tx_t *tx)
1510 {
1511         dsl_dataset_t *ds = arg1;
1512         struct recvendsyncarg *resa = arg2;
1513
1514         dsl_dataset_snapshot_sync(ds, resa->tosnap, tx);
1515
1516         /* set snapshot's creation time and guid */
1517         dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
1518         ds->ds_prev->ds_phys->ds_creation_time = resa->creation_time;
1519         ds->ds_prev->ds_phys->ds_guid = resa->toguid;
1520         ds->ds_prev->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
1521
1522         dmu_buf_will_dirty(ds->ds_dbuf, tx);
1523         ds->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
1524 }
1525
1526 static int
1527 dmu_recv_existing_end(dmu_recv_cookie_t *drc)
1528 {
1529         struct recvendsyncarg resa;
1530         dsl_dataset_t *ds = drc->drc_logical_ds;
1531         int err;
1532
1533         /*
1534          * XXX hack; seems the ds is still dirty and dsl_pool_zil_clean()
1535          * expects it to have a ds_user_ptr (and zil), but clone_swap()
1536          * can close it.
1537          */
1538         txg_wait_synced(ds->ds_dir->dd_pool, 0);
1539
1540         if (dsl_dataset_tryown(ds, FALSE, dmu_recv_tag)) {
1541                 err = dsl_dataset_clone_swap(drc->drc_real_ds, ds,
1542                     drc->drc_force);
1543                 if (err)
1544                         goto out;
1545         } else {
1546                 mutex_exit(&ds->ds_recvlock);
1547                 dsl_dataset_rele(ds, dmu_recv_tag);
1548                 (void) dsl_dataset_destroy(drc->drc_real_ds, dmu_recv_tag,
1549                     B_FALSE);
1550                 return (EBUSY);
1551         }
1552
1553         resa.creation_time = drc->drc_drrb->drr_creation_time;
1554         resa.toguid = drc->drc_drrb->drr_toguid;
1555         resa.tosnap = drc->drc_tosnap;
1556
1557         err = dsl_sync_task_do(ds->ds_dir->dd_pool,
1558             recv_end_check, recv_end_sync, ds, &resa, 3);
1559         if (err) {
1560                 /* swap back */
1561                 (void) dsl_dataset_clone_swap(drc->drc_real_ds, ds, B_TRUE);
1562         }
1563
1564 out:
1565         mutex_exit(&ds->ds_recvlock);
1566         dsl_dataset_disown(ds, dmu_recv_tag);
1567         (void) dsl_dataset_destroy(drc->drc_real_ds, dmu_recv_tag, B_FALSE);
1568         return (err);
1569 }
1570
1571 static int
1572 dmu_recv_new_end(dmu_recv_cookie_t *drc)
1573 {
1574         struct recvendsyncarg resa;
1575         dsl_dataset_t *ds = drc->drc_logical_ds;
1576         int err;
1577
1578         /*
1579          * XXX hack; seems the ds is still dirty and dsl_pool_zil_clean()
1580          * expects it to have a ds_user_ptr (and zil), but clone_swap()
1581          * can close it.
1582          */
1583         txg_wait_synced(ds->ds_dir->dd_pool, 0);
1584
1585         resa.creation_time = drc->drc_drrb->drr_creation_time;
1586         resa.toguid = drc->drc_drrb->drr_toguid;
1587         resa.tosnap = drc->drc_tosnap;
1588
1589         err = dsl_sync_task_do(ds->ds_dir->dd_pool,
1590             recv_end_check, recv_end_sync, ds, &resa, 3);
1591         if (err) {
1592                 /* clean up the fs we just recv'd into */
1593                 (void) dsl_dataset_destroy(ds, dmu_recv_tag, B_FALSE);
1594         } else {
1595                 /* release the hold from dmu_recv_begin */
1596                 dsl_dataset_disown(ds, dmu_recv_tag);
1597         }
1598         return (err);
1599 }
1600
1601 int
1602 dmu_recv_end(dmu_recv_cookie_t *drc)
1603 {
1604         if (drc->drc_logical_ds != drc->drc_real_ds)
1605                 return (dmu_recv_existing_end(drc));
1606         else
1607                 return (dmu_recv_new_end(drc));
1608 }