module/zfs/dmu_send.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2011 by Delphix. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011 by Delphix. All rights reserved.
  26  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  27  */
  28
  29 #include <sys/dmu.h>
  30 #include <sys/dmu_impl.h>
  31 #include <sys/dmu_tx.h>
  32 #include <sys/dbuf.h>
  33 #include <sys/dnode.h>
  34 #include <sys/zfs_context.h>
  35 #include <sys/dmu_objset.h>
  36 #include <sys/dmu_traverse.h>
  37 #include <sys/dsl_dataset.h>
  38 #include <sys/dsl_dir.h>
  39 #include <sys/dsl_prop.h>
  40 #include <sys/dsl_pool.h>
  41 #include <sys/dsl_synctask.h>
  42 #include <sys/spa_impl.h>
  43 #include <sys/zfs_ioctl.h>
  44 #include <sys/zap.h>
  45 #include <sys/zio_checksum.h>
  46 #include <sys/zfs_znode.h>
  47 #include <zfs_fletcher.h>
  48 #include <sys/avl.h>
  49 #include <sys/ddt.h>
  50 #include <sys/zfs_onexit.h>
  51
  52 /* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */
  53 int zfs_send_corrupt_data = B_FALSE;
  54
  55 static char *dmu_recv_tag = "dmu_recv_tag";
  56
  57 typedef struct dump_bytes_io {
  58         dmu_sendarg_t   *dbi_dsp;
  59         void            *dbi_buf;
  60         int             dbi_len;
  61 } dump_bytes_io_t;
  62
  63 static void
  64 dump_bytes_strategy(void *arg)
  65 {
  66         dump_bytes_io_t *dbi = (dump_bytes_io_t *)arg;
  67         dmu_sendarg_t *dsp = dbi->dbi_dsp;
  68         dsl_dataset_t *ds = dsp->dsa_os->os_dsl_dataset;
  69         ssize_t resid; /* have to get resid to get detailed errno */
  70         ASSERT0(dbi->dbi_len % 8);
  71
  72         fletcher_4_incremental_native(dbi->dbi_buf, dbi->dbi_len, &dsp->dsa_zc);
  73         dsp->dsa_err = vn_rdwr(UIO_WRITE, dsp->dsa_vp,
  74             (caddr_t)dbi->dbi_buf, dbi->dbi_len,
  75             0, UIO_SYSSPACE, FAPPEND, RLIM64_INFINITY, CRED(), &resid);
  76
  77         mutex_enter(&ds->ds_sendstream_lock);
  78         *dsp->dsa_off += dbi->dbi_len;
  79         mutex_exit(&ds->ds_sendstream_lock);
  80 }
  81
  82 static int
  83 dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
  84 {
  85         dump_bytes_io_t dbi;
  86
  87         dbi.dbi_dsp = dsp;
  88         dbi.dbi_buf = buf;
  89         dbi.dbi_len = len;
  90
  91         /*
  92          * The vn_rdwr() call is performed in a taskq to ensure that there is
  93          * always enough stack space to write safely to the target filesystem.
  94          * The ZIO_TYPE_FREE threads are used because there can be a lot of
  95          * them and they are used in vdev_file.c for a similar purpose.
  96          */
  97         spa_taskq_dispatch_sync(dmu_objset_spa(dsp->dsa_os), ZIO_TYPE_FREE,
  98             ZIO_TASKQ_ISSUE, dump_bytes_strategy, &dbi, TQ_SLEEP);
  99
 100         return (dsp->dsa_err);
 101 }
 102
 103 static int
 104 dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
 105     uint64_t length)
 106 {
 107         struct drr_free *drrf = &(dsp->dsa_drr->drr_u.drr_free);
 108
 109         if (length != -1ULL && offset + length < offset)
 110                 length = -1ULL;
 111
 112         /*
 113          * If there is a pending op, but it's not PENDING_FREE, push it out,
 114          * since free block aggregation can only be done for blocks of the
 115          * same type (i.e., DRR_FREE records can only be aggregated with
 116          * other DRR_FREE records.  DRR_FREEOBJECTS records can only be
 117          * aggregated with other DRR_FREEOBJECTS records.
 118          */
 119         if (dsp->dsa_pending_op != PENDING_NONE &&
 120             dsp->dsa_pending_op != PENDING_FREE) {
 121                 if (dump_bytes(dsp, dsp->dsa_drr,
 122                     sizeof (dmu_replay_record_t)) != 0)
 123                         return (EINTR);
 124                 dsp->dsa_pending_op = PENDING_NONE;
 125         }
 126
 127         if (dsp->dsa_pending_op == PENDING_FREE) {
 128                 /*
 129                  * There should never be a PENDING_FREE if length is -1
 130                  * (because dump_dnode is the only place where this
 131                  * function is called with a -1, and only after flushing
 132                  * any pending record).
 133                  */
 134                 ASSERT(length != -1ULL);
 135                 /*
 136                  * Check to see whether this free block can be aggregated
 137                  * with pending one.
 138                  */
 139                 if (drrf->drr_object == object && drrf->drr_offset +
 140                     drrf->drr_length == offset) {
 141                         drrf->drr_length += length;
 142                         return (0);
 143                 } else {
 144                         /* not a continuation.  Push out pending record */
 145                         if (dump_bytes(dsp, dsp->dsa_drr,
 146                             sizeof (dmu_replay_record_t)) != 0)
 147                                 return (EINTR);
 148                         dsp->dsa_pending_op = PENDING_NONE;
 149                 }
 150         }
 151         /* create a FREE record and make it pending */
 152         bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
 153         dsp->dsa_drr->drr_type = DRR_FREE;
 154         drrf->drr_object = object;
 155         drrf->drr_offset = offset;
 156         drrf->drr_length = length;
 157         drrf->drr_toguid = dsp->dsa_toguid;
 158         if (length == -1ULL) {
 159                 if (dump_bytes(dsp, dsp->dsa_drr,
 160                     sizeof (dmu_replay_record_t)) != 0)
 161                         return (EINTR);
 162         } else {
 163                 dsp->dsa_pending_op = PENDING_FREE;
 164         }
 165
 166         return (0);
 167 }
 168
 169 static int
 170 dump_data(dmu_sendarg_t *dsp, dmu_object_type_t type,
 171     uint64_t object, uint64_t offset, int blksz, const blkptr_t *bp, void *data)
 172 {
 173         struct drr_write *drrw = &(dsp->dsa_drr->drr_u.drr_write);
 174
 175
 176         /*
 177          * If there is any kind of pending aggregation (currently either
 178          * a grouping of free objects or free blocks), push it out to
 179          * the stream, since aggregation can't be done across operations
 180          * of different types.
 181          */
 182         if (dsp->dsa_pending_op != PENDING_NONE) {
 183                 if (dump_bytes(dsp, dsp->dsa_drr,
 184                     sizeof (dmu_replay_record_t)) != 0)
 185                         return (EINTR);
 186                 dsp->dsa_pending_op = PENDING_NONE;
 187         }
 188         /* write a DATA record */
 189         bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
 190         dsp->dsa_drr->drr_type = DRR_WRITE;
 191         drrw->drr_object = object;
 192         drrw->drr_type = type;
 193         drrw->drr_offset = offset;
 194         drrw->drr_length = blksz;
 195         drrw->drr_toguid = dsp->dsa_toguid;
 196         drrw->drr_checksumtype = BP_GET_CHECKSUM(bp);
 197         if (zio_checksum_table[drrw->drr_checksumtype].ci_dedup)
 198                 drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP;
 199         DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp));
 200         DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp));
 201         DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp));
 202         drrw->drr_key.ddk_cksum = bp->blk_cksum;
 203
 204         if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
 205                 return (EINTR);
 206         if (dump_bytes(dsp, data, blksz) != 0)
 207                 return (EINTR);
 208         return (0);
 209 }
 210
 211 static int
 212 dump_spill(dmu_sendarg_t *dsp, uint64_t object, int blksz, void *data)
 213 {
 214         struct drr_spill *drrs = &(dsp->dsa_drr->drr_u.drr_spill);
 215
 216         if (dsp->dsa_pending_op != PENDING_NONE) {
 217                 if (dump_bytes(dsp, dsp->dsa_drr,
 218                     sizeof (dmu_replay_record_t)) != 0)
 219                         return (EINTR);
 220                 dsp->dsa_pending_op = PENDING_NONE;
 221         }
 222
 223         /* write a SPILL record */
 224         bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
 225         dsp->dsa_drr->drr_type = DRR_SPILL;
 226         drrs->drr_object = object;
 227         drrs->drr_length = blksz;
 228         drrs->drr_toguid = dsp->dsa_toguid;
 229
 230         if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)))
 231                 return (EINTR);
 232         if (dump_bytes(dsp, data, blksz))
 233                 return (EINTR);
 234         return (0);
 235 }
 236
 237 static int
 238 dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs)
 239 {
 240         struct drr_freeobjects *drrfo = &(dsp->dsa_drr->drr_u.drr_freeobjects);
 241
 242         /*
 243          * If there is a pending op, but it's not PENDING_FREEOBJECTS,
 244          * push it out, since free block aggregation can only be done for
 245          * blocks of the same type (i.e., DRR_FREE records can only be
 246          * aggregated with other DRR_FREE records.  DRR_FREEOBJECTS records
 247          * can only be aggregated with other DRR_FREEOBJECTS records.
 248          */
 249         if (dsp->dsa_pending_op != PENDING_NONE &&
 250             dsp->dsa_pending_op != PENDING_FREEOBJECTS) {
 251                 if (dump_bytes(dsp, dsp->dsa_drr,
 252                     sizeof (dmu_replay_record_t)) != 0)
 253                         return (EINTR);
 254                 dsp->dsa_pending_op = PENDING_NONE;
 255         }
 256         if (dsp->dsa_pending_op == PENDING_FREEOBJECTS) {
 257                 /*
 258                  * See whether this free object array can be aggregated
 259                  * with pending one
 260                  */
 261                 if (drrfo->drr_firstobj + drrfo->drr_numobjs == firstobj) {
 262                         drrfo->drr_numobjs += numobjs;
 263                         return (0);
 264                 } else {
 265                         /* can't be aggregated.  Push out pending record */
 266                         if (dump_bytes(dsp, dsp->dsa_drr,
 267                             sizeof (dmu_replay_record_t)) != 0)
 268                                 return (EINTR);
 269                         dsp->dsa_pending_op = PENDING_NONE;
 270                 }
 271         }
 272
 273         /* write a FREEOBJECTS record */
 274         bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
 275         dsp->dsa_drr->drr_type = DRR_FREEOBJECTS;
 276         drrfo->drr_firstobj = firstobj;
 277         drrfo->drr_numobjs = numobjs;
 278         drrfo->drr_toguid = dsp->dsa_toguid;
 279
 280         dsp->dsa_pending_op = PENDING_FREEOBJECTS;
 281
 282         return (0);
 283 }
 284
 285 static int
 286 dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
 287 {
 288         struct drr_object *drro = &(dsp->dsa_drr->drr_u.drr_object);
 289
 290         if (dnp == NULL || dnp->dn_type == DMU_OT_NONE)
 291                 return (dump_freeobjects(dsp, object, 1));
 292
 293         if (dsp->dsa_pending_op != PENDING_NONE) {
 294                 if (dump_bytes(dsp, dsp->dsa_drr,
 295                     sizeof (dmu_replay_record_t)) != 0)
 296                         return (EINTR);
 297                 dsp->dsa_pending_op = PENDING_NONE;
 298         }
 299
 300         /* write an OBJECT record */
 301         bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
 302         dsp->dsa_drr->drr_type = DRR_OBJECT;
 303         drro->drr_object = object;
 304         drro->drr_type = dnp->dn_type;
 305         drro->drr_bonustype = dnp->dn_bonustype;
 306         drro->drr_blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
 307         drro->drr_bonuslen = dnp->dn_bonuslen;
 308         drro->drr_checksumtype = dnp->dn_checksum;
 309         drro->drr_compress = dnp->dn_compress;
 310         drro->drr_toguid = dsp->dsa_toguid;
 311
 312         if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
 313                 return (EINTR);
 314
 315         if (dump_bytes(dsp, DN_BONUS(dnp), P2ROUNDUP(dnp->dn_bonuslen, 8)) != 0)
 316                 return (EINTR);
 317
 318         /* free anything past the end of the file */
 319         if (dump_free(dsp, object, (dnp->dn_maxblkid + 1) *
 320             (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT), -1ULL))
 321                 return (EINTR);
 322         if (dsp->dsa_err)
 323                 return (EINTR);
 324         return (0);
 325 }
 326
 327 #define BP_SPAN(dnp, level) \
 328         (((uint64_t)dnp->dn_datablkszsec) << (SPA_MINBLOCKSHIFT + \
 329         (level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)))
 330
 331 /* ARGSUSED */
 332 static int
 333 backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, arc_buf_t *pbuf,
 334     const zbookmark_t *zb, const dnode_phys_t *dnp, void *arg)
 335 {
 336         dmu_sendarg_t *dsp = arg;
 337         dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE;
 338         int err = 0;
 339
 340         if (issig(JUSTLOOKING) && issig(FORREAL))
 341                 return (EINTR);
 342
 343         if (zb->zb_object != DMU_META_DNODE_OBJECT &&
 344             DMU_OBJECT_IS_SPECIAL(zb->zb_object)) {
 345                 return (0);
 346         } else if (bp == NULL && zb->zb_object == DMU_META_DNODE_OBJECT) {
 347                 uint64_t span = BP_SPAN(dnp, zb->zb_level);
 348                 uint64_t dnobj = (zb->zb_blkid * span) >> DNODE_SHIFT;
 349                 err = dump_freeobjects(dsp, dnobj, span >> DNODE_SHIFT);
 350         } else if (bp == NULL) {
 351                 uint64_t span = BP_SPAN(dnp, zb->zb_level);
 352                 err = dump_free(dsp, zb->zb_object, zb->zb_blkid * span, span);
 353         } else if (zb->zb_level > 0 || type == DMU_OT_OBJSET) {
 354                 return (0);
 355         } else if (type == DMU_OT_DNODE) {
 356                 dnode_phys_t *blk;
 357                 int i;
 358                 int blksz = BP_GET_LSIZE(bp);
 359                 uint32_t aflags = ARC_WAIT;
 360                 arc_buf_t *abuf;
 361
 362                 if (dsl_read(NULL, spa, bp, pbuf,
 363                     arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 364                     ZIO_FLAG_CANFAIL, &aflags, zb) != 0)
 365                         return (EIO);
 366
 367                 blk = abuf->b_data;
 368                 for (i = 0; i < blksz >> DNODE_SHIFT; i++) {
 369                         uint64_t dnobj = (zb->zb_blkid <<
 370                             (DNODE_BLOCK_SHIFT - DNODE_SHIFT)) + i;
 371                         err = dump_dnode(dsp, dnobj, blk+i);
 372                         if (err)
 373                                 break;
 374                 }
 375                 (void) arc_buf_remove_ref(abuf, &abuf);
 376         } else if (type == DMU_OT_SA) {
 377                 uint32_t aflags = ARC_WAIT;
 378                 arc_buf_t *abuf;
 379                 int blksz = BP_GET_LSIZE(bp);
 380
 381                 if (arc_read_nolock(NULL, spa, bp,
 382                     arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 383                     ZIO_FLAG_CANFAIL, &aflags, zb) != 0)
 384                         return (EIO);
 385
 386                 err = dump_spill(dsp, zb->zb_object, blksz, abuf->b_data);
 387                 (void) arc_buf_remove_ref(abuf, &abuf);
 388         } else { /* it's a level-0 block of a regular object */
 389                 uint32_t aflags = ARC_WAIT;
 390                 arc_buf_t *abuf;
 391                 int blksz = BP_GET_LSIZE(bp);
 392
 393                 if (dsl_read(NULL, spa, bp, pbuf,
 394                     arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 395                     ZIO_FLAG_CANFAIL, &aflags, zb) != 0) {
 396                         if (zfs_send_corrupt_data) {
 397                                 uint64_t *ptr;
 398                                 /* Send a block filled with 0x"zfs badd bloc" */
 399                                 abuf = arc_buf_alloc(spa, blksz, &abuf,
 400                                     ARC_BUFC_DATA);
 401                                 for (ptr = abuf->b_data;
 402                                     (char *)ptr < (char *)abuf->b_data + blksz;
 403                                     ptr++)
 404                                         *ptr = 0x2f5baddb10cULL;
 405                         } else {
 406                                 return (EIO);
 407                         }
 408                 }
 409
 410                 err = dump_data(dsp, type, zb->zb_object, zb->zb_blkid * blksz,
 411                     blksz, bp, abuf->b_data);
 412                 (void) arc_buf_remove_ref(abuf, &abuf);
 413         }
 414
 415         ASSERT(err == 0 || err == EINTR);
 416         return (err);
 417 }
 418
 419 int
 420 dmu_send(objset_t *tosnap, objset_t *fromsnap, boolean_t fromorigin,
 421     int outfd, vnode_t *vp, offset_t *off)
 422 {
 423         dsl_dataset_t *ds = tosnap->os_dsl_dataset;
 424         dsl_dataset_t *fromds = fromsnap ? fromsnap->os_dsl_dataset : NULL;
 425         dmu_replay_record_t *drr;
 426         dmu_sendarg_t *dsp;
 427         int err;
 428         uint64_t fromtxg = 0;
 429
 430         /* tosnap must be a snapshot */
 431         if (ds->ds_phys->ds_next_snap_obj == 0)
 432                 return (EINVAL);
 433
 434         /* fromsnap must be an earlier snapshot from the same fs as tosnap */
 435         if (fromds && (ds->ds_dir != fromds->ds_dir ||
 436             fromds->ds_phys->ds_creation_txg >= ds->ds_phys->ds_creation_txg))
 437                 return (EXDEV);
 438
 439         if (fromorigin) {
 440                 dsl_pool_t *dp = ds->ds_dir->dd_pool;
 441
 442                 if (fromsnap)
 443                         return (EINVAL);
 444
 445                 if (dsl_dir_is_clone(ds->ds_dir)) {
 446                         rw_enter(&dp->dp_config_rwlock, RW_READER);
 447                         err = dsl_dataset_hold_obj(dp,
 448                             ds->ds_dir->dd_phys->dd_origin_obj, FTAG, &fromds);
 449                         rw_exit(&dp->dp_config_rwlock);
 450                         if (err)
 451                                 return (err);
 452                 } else {
 453                         fromorigin = B_FALSE;
 454                 }
 455         }
 456
 457
 458         drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
 459         drr->drr_type = DRR_BEGIN;
 460         drr->drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
 461         DMU_SET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo,
 462             DMU_SUBSTREAM);
 463
 464 #ifdef _KERNEL
 465         if (dmu_objset_type(tosnap) == DMU_OST_ZFS) {
 466                 uint64_t version;
 467                 if (zfs_get_zplprop(tosnap, ZFS_PROP_VERSION, &version) != 0) {
 468                         kmem_free(drr, sizeof (dmu_replay_record_t));
 469                         return (EINVAL);
 470                 }
 471                 if (version == ZPL_VERSION_SA) {
 472                         DMU_SET_FEATUREFLAGS(
 473                             drr->drr_u.drr_begin.drr_versioninfo,
 474                             DMU_BACKUP_FEATURE_SA_SPILL);
 475                 }
 476         }
 477 #endif
 478
 479         drr->drr_u.drr_begin.drr_creation_time =
 480             ds->ds_phys->ds_creation_time;
 481         drr->drr_u.drr_begin.drr_type = tosnap->os_phys->os_type;
 482         if (fromorigin)
 483                 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CLONE;
 484         drr->drr_u.drr_begin.drr_toguid = ds->ds_phys->ds_guid;
 485         if (ds->ds_phys->ds_flags & DS_FLAG_CI_DATASET)
 486                 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA;
 487
 488         if (fromds)
 489                 drr->drr_u.drr_begin.drr_fromguid = fromds->ds_phys->ds_guid;
 490         dsl_dataset_name(ds, drr->drr_u.drr_begin.drr_toname);
 491
 492         if (fromds)
 493                 fromtxg = fromds->ds_phys->ds_creation_txg;
 494         if (fromorigin)
 495                 dsl_dataset_rele(fromds, FTAG);
 496
 497         dsp = kmem_zalloc(sizeof (dmu_sendarg_t), KM_SLEEP);
 498
 499         dsp->dsa_drr = drr;
 500         dsp->dsa_vp = vp;
 501         dsp->dsa_outfd = outfd;
 502         dsp->dsa_proc = curproc;
 503         dsp->dsa_os = tosnap;
 504         dsp->dsa_off = off;
 505         dsp->dsa_toguid = ds->ds_phys->ds_guid;
 506         ZIO_SET_CHECKSUM(&dsp->dsa_zc, 0, 0, 0, 0);
 507         dsp->dsa_pending_op = PENDING_NONE;
 508
 509         mutex_enter(&ds->ds_sendstream_lock);
 510         list_insert_head(&ds->ds_sendstreams, dsp);
 511         mutex_exit(&ds->ds_sendstream_lock);
 512
 513         if (dump_bytes(dsp, drr, sizeof (dmu_replay_record_t)) != 0) {
 514                 err = dsp->dsa_err;
 515                 goto out;
 516         }
 517
 518         err = traverse_dataset(ds, fromtxg, TRAVERSE_PRE | TRAVERSE_PREFETCH,
 519             backup_cb, dsp);
 520
 521         if (dsp->dsa_pending_op != PENDING_NONE)
 522                 if (dump_bytes(dsp, drr, sizeof (dmu_replay_record_t)) != 0)
 523                         err = EINTR;
 524
 525         if (err) {
 526                 if (err == EINTR && dsp->dsa_err)
 527                         err = dsp->dsa_err;
 528                 goto out;
 529         }
 530
 531         bzero(drr, sizeof (dmu_replay_record_t));
 532         drr->drr_type = DRR_END;
 533         drr->drr_u.drr_end.drr_checksum = dsp->dsa_zc;
 534         drr->drr_u.drr_end.drr_toguid = dsp->dsa_toguid;
 535
 536         if (dump_bytes(dsp, drr, sizeof (dmu_replay_record_t)) != 0) {
 537                 err = dsp->dsa_err;
 538                 goto out;
 539         }
 540
 541 out:
 542         mutex_enter(&ds->ds_sendstream_lock);
 543         list_remove(&ds->ds_sendstreams, dsp);
 544         mutex_exit(&ds->ds_sendstream_lock);
 545
 546         kmem_free(drr, sizeof (dmu_replay_record_t));
 547         kmem_free(dsp, sizeof (dmu_sendarg_t));
 548
 549         return (err);
 550 }
 551
 552 int
 553 dmu_send_estimate(objset_t *tosnap, objset_t *fromsnap, boolean_t fromorigin,
 554     uint64_t *sizep)
 555 {
 556         dsl_dataset_t *ds = tosnap->os_dsl_dataset;
 557         dsl_dataset_t *fromds = fromsnap ? fromsnap->os_dsl_dataset : NULL;
 558         dsl_pool_t *dp = ds->ds_dir->dd_pool;
 559         int err;
 560         uint64_t size, recordsize;
 561
 562         /* tosnap must be a snapshot */
 563         if (ds->ds_phys->ds_next_snap_obj == 0)
 564                 return (EINVAL);
 565
 566         /* fromsnap must be an earlier snapshot from the same fs as tosnap */
 567         if (fromds && (ds->ds_dir != fromds->ds_dir ||
 568             fromds->ds_phys->ds_creation_txg >= ds->ds_phys->ds_creation_txg))
 569                 return (EXDEV);
 570
 571         if (fromorigin) {
 572                 if (fromsnap)
 573                         return (EINVAL);
 574
 575                 if (dsl_dir_is_clone(ds->ds_dir)) {
 576                         rw_enter(&dp->dp_config_rwlock, RW_READER);
 577                         err = dsl_dataset_hold_obj(dp,
 578                             ds->ds_dir->dd_phys->dd_origin_obj, FTAG, &fromds);
 579                         rw_exit(&dp->dp_config_rwlock);
 580                         if (err)
 581                                 return (err);
 582                 } else {
 583                         fromorigin = B_FALSE;
 584                 }
 585         }
 586
 587         /* Get uncompressed size estimate of changed data. */
 588         if (fromds == NULL) {
 589                 size = ds->ds_phys->ds_uncompressed_bytes;
 590         } else {
 591                 uint64_t used, comp;
 592                 err = dsl_dataset_space_written(fromds, ds,
 593                     &used, &comp, &size);
 594                 if (fromorigin)
 595                         dsl_dataset_rele(fromds, FTAG);
 596                 if (err)
 597                         return (err);
 598         }
 599
 600         /*
 601          * Assume that space (both on-disk and in-stream) is dominated by
 602          * data.  We will adjust for indirect blocks and the copies property,
 603          * but ignore per-object space used (eg, dnodes and DRR_OBJECT records).
 604          */
 605
 606         /*
 607          * Subtract out approximate space used by indirect blocks.
 608          * Assume most space is used by data blocks (non-indirect, non-dnode).
 609          * Assume all blocks are recordsize.  Assume ditto blocks and
 610          * internal fragmentation counter out compression.
 611          *
 612          * Therefore, space used by indirect blocks is sizeof(blkptr_t) per
 613          * block, which we observe in practice.
 614          */
 615         rw_enter(&dp->dp_config_rwlock, RW_READER);
 616         err = dsl_prop_get_ds(ds, "recordsize",
 617             sizeof (recordsize), 1, &recordsize, NULL);
 618         rw_exit(&dp->dp_config_rwlock);
 619         if (err)
 620                 return (err);
 621         size -= size / recordsize * sizeof (blkptr_t);
 622
 623         /* Add in the space for the record associated with each block. */
 624         size += size / recordsize * sizeof (dmu_replay_record_t);
 625
 626         *sizep = size;
 627
 628         return (0);
 629 }
 630
 631 struct recvbeginsyncarg {
 632         const char *tofs;
 633         const char *tosnap;
 634         dsl_dataset_t *origin;
 635         uint64_t fromguid;
 636         dmu_objset_type_t type;
 637         void *tag;
 638         boolean_t force;
 639         uint64_t dsflags;
 640         char clonelastname[MAXNAMELEN];
 641         dsl_dataset_t *ds; /* the ds to recv into; returned from the syncfunc */
 642         cred_t *cr;
 643 };
 644
 645 /* ARGSUSED */
 646 static int
 647 recv_new_check(void *arg1, void *arg2, dmu_tx_t *tx)
 648 {
 649         dsl_dir_t *dd = arg1;
 650         struct recvbeginsyncarg *rbsa = arg2;
 651         objset_t *mos = dd->dd_pool->dp_meta_objset;
 652         uint64_t val;
 653         int err;
 654
 655         err = zap_lookup(mos, dd->dd_phys->dd_child_dir_zapobj,
 656             strrchr(rbsa->tofs, '/') + 1, sizeof (uint64_t), 1, &val);
 657
 658         if (err != ENOENT)
 659                 return (err ? err : EEXIST);
 660
 661         if (rbsa->origin) {
 662                 /* make sure it's a snap in the same pool */
 663                 if (rbsa->origin->ds_dir->dd_pool != dd->dd_pool)
 664                         return (EXDEV);
 665                 if (!dsl_dataset_is_snapshot(rbsa->origin))
 666                         return (EINVAL);
 667                 if (rbsa->origin->ds_phys->ds_guid != rbsa->fromguid)
 668                         return (ENODEV);
 669         }
 670
 671         return (0);
 672 }
 673
 674 static void
 675 recv_new_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 676 {
 677         dsl_dir_t *dd = arg1;
 678         struct recvbeginsyncarg *rbsa = arg2;
 679         uint64_t flags = DS_FLAG_INCONSISTENT | rbsa->dsflags;
 680         uint64_t dsobj;
 681
 682         /* Create and open new dataset. */
 683         dsobj = dsl_dataset_create_sync(dd, strrchr(rbsa->tofs, '/') + 1,
 684             rbsa->origin, flags, rbsa->cr, tx);
 685         VERIFY(0 == dsl_dataset_own_obj(dd->dd_pool, dsobj,
 686             B_TRUE, dmu_recv_tag, &rbsa->ds));
 687
 688         if (rbsa->origin == NULL) {
 689                 (void) dmu_objset_create_impl(dd->dd_pool->dp_spa,
 690                     rbsa->ds, &rbsa->ds->ds_phys->ds_bp, rbsa->type, tx);
 691         }
 692
 693         spa_history_log_internal(LOG_DS_REPLAY_FULL_SYNC,
 694             dd->dd_pool->dp_spa, tx, "dataset = %lld", dsobj);
 695 }
 696
 697 /* ARGSUSED */
 698 static int
 699 recv_existing_check(void *arg1, void *arg2, dmu_tx_t *tx)
 700 {
 701         dsl_dataset_t *ds = arg1;
 702         struct recvbeginsyncarg *rbsa = arg2;
 703         int err;
 704         uint64_t val;
 705
 706         /* must not have any changes since most recent snapshot */
 707         if (!rbsa->force && dsl_dataset_modified_since_lastsnap(ds))
 708                 return (ETXTBSY);
 709
 710         /* new snapshot name must not exist */
 711         err = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset,
 712             ds->ds_phys->ds_snapnames_zapobj, rbsa->tosnap, 8, 1, &val);
 713         if (err == 0)
 714                 return (EEXIST);
 715         if (err != ENOENT)
 716                 return (err);
 717
 718         if (rbsa->fromguid) {
 719                 /* if incremental, most recent snapshot must match fromguid */
 720                 if (ds->ds_prev == NULL)
 721                         return (ENODEV);
 722
 723                 /*
 724                  * most recent snapshot must match fromguid, or there are no
 725                  * changes since the fromguid one
 726                  */
 727                 if (ds->ds_prev->ds_phys->ds_guid != rbsa->fromguid) {
 728                         uint64_t birth = ds->ds_prev->ds_phys->ds_bp.blk_birth;
 729                         uint64_t obj = ds->ds_prev->ds_phys->ds_prev_snap_obj;
 730                         while (obj != 0) {
 731                                 dsl_dataset_t *snap;
 732                                 err = dsl_dataset_hold_obj(ds->ds_dir->dd_pool,
 733                                     obj, FTAG, &snap);
 734                                 if (err)
 735                                         return (ENODEV);
 736                                 if (snap->ds_phys->ds_creation_txg < birth) {
 737                                         dsl_dataset_rele(snap, FTAG);
 738                                         return (ENODEV);
 739                                 }
 740                                 if (snap->ds_phys->ds_guid == rbsa->fromguid) {
 741                                         dsl_dataset_rele(snap, FTAG);
 742                                         break; /* it's ok */
 743                                 }
 744                                 obj = snap->ds_phys->ds_prev_snap_obj;
 745                                 dsl_dataset_rele(snap, FTAG);
 746                         }
 747                         if (obj == 0)
 748                                 return (ENODEV);
 749                 }
 750         } else {
 751                 /* if full, most recent snapshot must be $ORIGIN */
 752                 if (ds->ds_phys->ds_prev_snap_txg >= TXG_INITIAL)
 753                         return (ENODEV);
 754         }
 755
 756         /* temporary clone name must not exist */
 757         err = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset,
 758             ds->ds_dir->dd_phys->dd_child_dir_zapobj,
 759             rbsa->clonelastname, 8, 1, &val);
 760         if (err == 0)
 761                 return (EEXIST);
 762         if (err != ENOENT)
 763                 return (err);
 764
 765         return (0);
 766 }
 767
 768 /* ARGSUSED */
 769 static void
 770 recv_existing_sync(void *arg1, void *arg2, dmu_tx_t *tx)
 771 {
 772         dsl_dataset_t *ohds = arg1;
 773         struct recvbeginsyncarg *rbsa = arg2;
 774         dsl_pool_t *dp = ohds->ds_dir->dd_pool;
 775         dsl_dataset_t *cds;
 776         uint64_t flags = DS_FLAG_INCONSISTENT | rbsa->dsflags;
 777         uint64_t dsobj;
 778
 779         /* create and open the temporary clone */
 780         dsobj = dsl_dataset_create_sync(ohds->ds_dir, rbsa->clonelastname,
 781             ohds->ds_prev, flags, rbsa->cr, tx);
 782         VERIFY(0 == dsl_dataset_own_obj(dp, dsobj, B_TRUE, dmu_recv_tag, &cds));
 783
 784         /*
 785          * If we actually created a non-clone, we need to create the
 786          * objset in our new dataset.
 787          */
 788         if (BP_IS_HOLE(dsl_dataset_get_blkptr(cds))) {
 789                 (void) dmu_objset_create_impl(dp->dp_spa,
 790                     cds, dsl_dataset_get_blkptr(cds), rbsa->type, tx);
 791         }
 792
 793         rbsa->ds = cds;
 794
 795         spa_history_log_internal(LOG_DS_REPLAY_INC_SYNC,
 796             dp->dp_spa, tx, "dataset = %lld", dsobj);
 797 }
 798
 799 static boolean_t
 800 dmu_recv_verify_features(dsl_dataset_t *ds, struct drr_begin *drrb)
 801 {
 802         int featureflags;
 803
 804         featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
 805
 806         /* Verify pool version supports SA if SA_SPILL feature set */
 807         return ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
 808             (spa_version(dsl_dataset_get_spa(ds)) < SPA_VERSION_SA));
 809 }
 810
 811 /*
 812  * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin()
 813  * succeeds; otherwise we will leak the holds on the datasets.
 814  */
 815 int
 816 dmu_recv_begin(char *tofs, char *tosnap, char *top_ds, struct drr_begin *drrb,
 817     boolean_t force, objset_t *origin, dmu_recv_cookie_t *drc)
 818 {
 819         int err = 0;
 820         boolean_t byteswap;
 821         struct recvbeginsyncarg rbsa = { 0 };
 822         uint64_t versioninfo;
 823         int flags;
 824         dsl_dataset_t *ds;
 825
 826         if (drrb->drr_magic == DMU_BACKUP_MAGIC)
 827                 byteswap = FALSE;
 828         else if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC))
 829                 byteswap = TRUE;
 830         else
 831                 return (EINVAL);
 832
 833         rbsa.tofs = tofs;
 834         rbsa.tosnap = tosnap;
 835         rbsa.origin = origin ? origin->os_dsl_dataset : NULL;
 836         rbsa.fromguid = drrb->drr_fromguid;
 837         rbsa.type = drrb->drr_type;
 838         rbsa.tag = FTAG;
 839         rbsa.dsflags = 0;
 840         rbsa.cr = CRED();
 841         versioninfo = drrb->drr_versioninfo;
 842         flags = drrb->drr_flags;
 843
 844         if (byteswap) {
 845                 rbsa.type = BSWAP_32(rbsa.type);
 846                 rbsa.fromguid = BSWAP_64(rbsa.fromguid);
 847                 versioninfo = BSWAP_64(versioninfo);
 848                 flags = BSWAP_32(flags);
 849         }
 850
 851         if (DMU_GET_STREAM_HDRTYPE(versioninfo) == DMU_COMPOUNDSTREAM ||
 852             rbsa.type >= DMU_OST_NUMTYPES ||
 853             ((flags & DRR_FLAG_CLONE) && origin == NULL))
 854                 return (EINVAL);
 855
 856         if (flags & DRR_FLAG_CI_DATA)
 857                 rbsa.dsflags = DS_FLAG_CI_DATASET;
 858
 859         bzero(drc, sizeof (dmu_recv_cookie_t));
 860         drc->drc_drrb = drrb;
 861         drc->drc_tosnap = tosnap;
 862         drc->drc_top_ds = top_ds;
 863         drc->drc_force = force;
 864
 865         /*
 866          * Process the begin in syncing context.
 867          */
 868
 869         /* open the dataset we are logically receiving into */
 870         err = dsl_dataset_hold(tofs, dmu_recv_tag, &ds);
 871         if (err == 0) {
 872                 if (dmu_recv_verify_features(ds, drrb)) {
 873                         dsl_dataset_rele(ds, dmu_recv_tag);
 874                         return (ENOTSUP);
 875                 }
 876                 /* target fs already exists; recv into temp clone */
 877
 878                 /* Can't recv a clone into an existing fs */
 879                 if (flags & DRR_FLAG_CLONE) {
 880                         dsl_dataset_rele(ds, dmu_recv_tag);
 881                         return (EINVAL);
 882                 }
 883
 884                 /* must not have an incremental recv already in progress */
 885                 if (!mutex_tryenter(&ds->ds_recvlock)) {
 886                         dsl_dataset_rele(ds, dmu_recv_tag);
 887                         return (EBUSY);
 888                 }
 889
 890                 /* tmp clone name is: tofs/%tosnap" */
 891                 (void) snprintf(rbsa.clonelastname, sizeof (rbsa.clonelastname),
 892                     "%%%s", tosnap);
 893                 rbsa.force = force;
 894                 err = dsl_sync_task_do(ds->ds_dir->dd_pool,
 895                     recv_existing_check, recv_existing_sync, ds, &rbsa, 5);
 896                 if (err) {
 897                         mutex_exit(&ds->ds_recvlock);
 898                         dsl_dataset_rele(ds, dmu_recv_tag);
 899                         return (err);
 900                 }
 901                 drc->drc_logical_ds = ds;
 902                 drc->drc_real_ds = rbsa.ds;
 903         } else if (err == ENOENT) {
 904                 /* target fs does not exist; must be a full backup or clone */
 905                 char *cp;
 906
 907                 /*
 908                  * If it's a non-clone incremental, we are missing the
 909                  * target fs, so fail the recv.
 910                  */
 911                 if (rbsa.fromguid && !(flags & DRR_FLAG_CLONE))
 912                         return (ENOENT);
 913
 914                 /* Open the parent of tofs */
 915                 cp = strrchr(tofs, '/');
 916                 *cp = '\0';
 917                 err = dsl_dataset_hold(tofs, FTAG, &ds);
 918                 *cp = '/';
 919                 if (err)
 920                         return (err);
 921
 922                 if (dmu_recv_verify_features(ds, drrb)) {
 923                         dsl_dataset_rele(ds, FTAG);
 924                         return (ENOTSUP);
 925                 }
 926
 927                 err = dsl_sync_task_do(ds->ds_dir->dd_pool,
 928                     recv_new_check, recv_new_sync, ds->ds_dir, &rbsa, 5);
 929                 dsl_dataset_rele(ds, FTAG);
 930                 if (err)
 931                         return (err);
 932                 drc->drc_logical_ds = drc->drc_real_ds = rbsa.ds;
 933                 drc->drc_newfs = B_TRUE;
 934         }
 935
 936         return (err);
 937 }
 938
 939 struct restorearg {
 940         int err;
 941         int byteswap;
 942         vnode_t *vp;
 943         char *buf;
 944         uint64_t voff;
 945         int bufsize; /* amount of memory allocated for buf */
 946         zio_cksum_t cksum;
 947         avl_tree_t *guid_to_ds_map;
 948 };
 949
 950 typedef struct guid_map_entry {
 951         uint64_t        guid;
 952         dsl_dataset_t   *gme_ds;
 953         avl_node_t      avlnode;
 954 } guid_map_entry_t;
 955
 956 static int
 957 guid_compare(const void *arg1, const void *arg2)
 958 {
 959         const guid_map_entry_t *gmep1 = arg1;
 960         const guid_map_entry_t *gmep2 = arg2;
 961
 962         if (gmep1->guid < gmep2->guid)
 963                 return (-1);
 964         else if (gmep1->guid > gmep2->guid)
 965                 return (1);
 966         return (0);
 967 }
 968
 969 static void
 970 free_guid_map_onexit(void *arg)
 971 {
 972         avl_tree_t *ca = arg;
 973         void *cookie = NULL;
 974         guid_map_entry_t *gmep;
 975
 976         while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) {
 977                 dsl_dataset_rele(gmep->gme_ds, ca);
 978                 kmem_free(gmep, sizeof (guid_map_entry_t));
 979         }
 980         avl_destroy(ca);
 981         kmem_free(ca, sizeof (avl_tree_t));
 982 }
 983
 984 static void *
 985 restore_read(struct restorearg *ra, int len)
 986 {
 987         void *rv;
 988         int done = 0;
 989
 990         /* some things will require 8-byte alignment, so everything must */
 991         ASSERT0(len % 8);
 992
 993         while (done < len) {
 994                 ssize_t resid;
 995
 996                 ra->err = vn_rdwr(UIO_READ, ra->vp,
 997                     (caddr_t)ra->buf + done, len - done,
 998                     ra->voff, UIO_SYSSPACE, FAPPEND,
 999                     RLIM64_INFINITY, CRED(), &resid);
1000
1001                 if (resid == len - done)
1002                         ra->err = EINVAL;
1003                 ra->voff += len - done - resid;
1004                 done = len - resid;
1005                 if (ra->err)
1006                         return (NULL);
1007         }
1008
1009         ASSERT3U(done, ==, len);
1010         rv = ra->buf;
1011         if (ra->byteswap)
1012                 fletcher_4_incremental_byteswap(rv, len, &ra->cksum);
1013         else
1014                 fletcher_4_incremental_native(rv, len, &ra->cksum);
1015         return (rv);
1016 }
1017
1018 noinline static void
1019 backup_byteswap(dmu_replay_record_t *drr)
1020 {
1021 #define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
1022 #define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
1023         drr->drr_type = BSWAP_32(drr->drr_type);
1024         drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);
1025         switch (drr->drr_type) {
1026         case DRR_BEGIN:
1027                 DO64(drr_begin.drr_magic);
1028                 DO64(drr_begin.drr_versioninfo);
1029                 DO64(drr_begin.drr_creation_time);
1030                 DO32(drr_begin.drr_type);
1031                 DO32(drr_begin.drr_flags);
1032                 DO64(drr_begin.drr_toguid);
1033                 DO64(drr_begin.drr_fromguid);
1034                 break;
1035         case DRR_OBJECT:
1036                 DO64(drr_object.drr_object);
1037                 /* DO64(drr_object.drr_allocation_txg); */
1038                 DO32(drr_object.drr_type);
1039                 DO32(drr_object.drr_bonustype);
1040                 DO32(drr_object.drr_blksz);
1041                 DO32(drr_object.drr_bonuslen);
1042                 DO64(drr_object.drr_toguid);
1043                 break;
1044         case DRR_FREEOBJECTS:
1045                 DO64(drr_freeobjects.drr_firstobj);
1046                 DO64(drr_freeobjects.drr_numobjs);
1047                 DO64(drr_freeobjects.drr_toguid);
1048                 break;
1049         case DRR_WRITE:
1050                 DO64(drr_write.drr_object);
1051                 DO32(drr_write.drr_type);
1052                 DO64(drr_write.drr_offset);
1053                 DO64(drr_write.drr_length);
1054                 DO64(drr_write.drr_toguid);
1055                 DO64(drr_write.drr_key.ddk_cksum.zc_word[0]);
1056                 DO64(drr_write.drr_key.ddk_cksum.zc_word[1]);
1057                 DO64(drr_write.drr_key.ddk_cksum.zc_word[2]);
1058                 DO64(drr_write.drr_key.ddk_cksum.zc_word[3]);
1059                 DO64(drr_write.drr_key.ddk_prop);
1060                 break;
1061         case DRR_WRITE_BYREF:
1062                 DO64(drr_write_byref.drr_object);
1063                 DO64(drr_write_byref.drr_offset);
1064                 DO64(drr_write_byref.drr_length);
1065                 DO64(drr_write_byref.drr_toguid);
1066                 DO64(drr_write_byref.drr_refguid);
1067                 DO64(drr_write_byref.drr_refobject);
1068                 DO64(drr_write_byref.drr_refoffset);
1069                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[0]);
1070                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[1]);
1071                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[2]);
1072                 DO64(drr_write_byref.drr_key.ddk_cksum.zc_word[3]);
1073                 DO64(drr_write_byref.drr_key.ddk_prop);
1074                 break;
1075         case DRR_FREE:
1076                 DO64(drr_free.drr_object);
1077                 DO64(drr_free.drr_offset);
1078                 DO64(drr_free.drr_length);
1079                 DO64(drr_free.drr_toguid);
1080                 break;
1081         case DRR_SPILL:
1082                 DO64(drr_spill.drr_object);
1083                 DO64(drr_spill.drr_length);
1084                 DO64(drr_spill.drr_toguid);
1085                 break;
1086         case DRR_END:
1087                 DO64(drr_end.drr_checksum.zc_word[0]);
1088                 DO64(drr_end.drr_checksum.zc_word[1]);
1089                 DO64(drr_end.drr_checksum.zc_word[2]);
1090                 DO64(drr_end.drr_checksum.zc_word[3]);
1091                 DO64(drr_end.drr_toguid);
1092                 break;
1093         default:
1094                 break;
1095         }
1096 #undef DO64
1097 #undef DO32
1098 }
1099
1100 noinline static int
1101 restore_object(struct restorearg *ra, objset_t *os, struct drr_object *drro)
1102 {
1103         int err;
1104         dmu_tx_t *tx;
1105         void *data = NULL;
1106
1107         if (drro->drr_type == DMU_OT_NONE ||
1108             !DMU_OT_IS_VALID(drro->drr_type) ||
1109             !DMU_OT_IS_VALID(drro->drr_bonustype) ||
1110             drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS ||
1111             drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
1112             P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
1113             drro->drr_blksz < SPA_MINBLOCKSIZE ||
1114             drro->drr_blksz > SPA_MAXBLOCKSIZE ||
1115             drro->drr_bonuslen > DN_MAX_BONUSLEN) {
1116                 return (EINVAL);
1117         }
1118
1119         err = dmu_object_info(os, drro->drr_object, NULL);
1120
1121         if (err != 0 && err != ENOENT)
1122                 return (EINVAL);
1123
1124         if (drro->drr_bonuslen) {
1125                 data = restore_read(ra, P2ROUNDUP(drro->drr_bonuslen, 8));
1126                 if (ra->err)
1127                         return (ra->err);
1128         }
1129
1130         if (err == ENOENT) {
1131                 /* currently free, want to be allocated */
1132                 tx = dmu_tx_create(os);
1133                 dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1134                 err = dmu_tx_assign(tx, TXG_WAIT);
1135                 if (err) {
1136                         dmu_tx_abort(tx);
1137                         return (err);
1138                 }
1139                 err = dmu_object_claim(os, drro->drr_object,
1140                     drro->drr_type, drro->drr_blksz,
1141                     drro->drr_bonustype, drro->drr_bonuslen, tx);
1142                 dmu_tx_commit(tx);
1143         } else {
1144                 /* currently allocated, want to be allocated */
1145                 err = dmu_object_reclaim(os, drro->drr_object,
1146                     drro->drr_type, drro->drr_blksz,
1147                     drro->drr_bonustype, drro->drr_bonuslen);
1148         }
1149         if (err) {
1150                 return (EINVAL);
1151         }
1152
1153         tx = dmu_tx_create(os);
1154         dmu_tx_hold_bonus(tx, drro->drr_object);
1155         err = dmu_tx_assign(tx, TXG_WAIT);
1156         if (err) {
1157                 dmu_tx_abort(tx);
1158                 return (err);
1159         }
1160
1161         dmu_object_set_checksum(os, drro->drr_object, drro->drr_checksumtype,
1162             tx);
1163         dmu_object_set_compress(os, drro->drr_object, drro->drr_compress, tx);
1164
1165         if (data != NULL) {
1166                 dmu_buf_t *db;
1167
1168                 VERIFY(0 == dmu_bonus_hold(os, drro->drr_object, FTAG, &db));
1169                 dmu_buf_will_dirty(db, tx);
1170
1171                 ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
1172                 bcopy(data, db->db_data, drro->drr_bonuslen);
1173                 if (ra->byteswap) {
1174                         dmu_object_byteswap_t byteswap =
1175                             DMU_OT_BYTESWAP(drro->drr_bonustype);
1176                         dmu_ot_byteswap[byteswap].ob_func(db->db_data,
1177                             drro->drr_bonuslen);
1178                 }
1179                 dmu_buf_rele(db, FTAG);
1180         }
1181         dmu_tx_commit(tx);
1182         return (0);
1183 }
1184
1185 /* ARGSUSED */
1186 noinline static int
1187 restore_freeobjects(struct restorearg *ra, objset_t *os,
1188     struct drr_freeobjects *drrfo)
1189 {
1190         uint64_t obj;
1191
1192         if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj)
1193                 return (EINVAL);
1194
1195         for (obj = drrfo->drr_firstobj;
1196             obj < drrfo->drr_firstobj + drrfo->drr_numobjs;
1197             (void) dmu_object_next(os, &obj, FALSE, 0)) {
1198                 int err;
1199
1200                 if (dmu_object_info(os, obj, NULL) != 0)
1201                         continue;
1202
1203                 err = dmu_free_object(os, obj);
1204                 if (err)
1205                         return (err);
1206         }
1207         return (0);
1208 }
1209
1210 noinline static int
1211 restore_write(struct restorearg *ra, objset_t *os,
1212     struct drr_write *drrw)
1213 {
1214         dmu_tx_t *tx;
1215         void *data;
1216         int err;
1217
1218         if (drrw->drr_offset + drrw->drr_length < drrw->drr_offset ||
1219             !DMU_OT_IS_VALID(drrw->drr_type))
1220                 return (EINVAL);
1221
1222         data = restore_read(ra, drrw->drr_length);
1223         if (data == NULL)
1224                 return (ra->err);
1225
1226         if (dmu_object_info(os, drrw->drr_object, NULL) != 0)
1227                 return (EINVAL);
1228
1229         tx = dmu_tx_create(os);
1230
1231         dmu_tx_hold_write(tx, drrw->drr_object,
1232             drrw->drr_offset, drrw->drr_length);
1233         err = dmu_tx_assign(tx, TXG_WAIT);
1234         if (err) {
1235                 dmu_tx_abort(tx);
1236                 return (err);
1237         }
1238         if (ra->byteswap) {
1239                 dmu_object_byteswap_t byteswap =
1240                     DMU_OT_BYTESWAP(drrw->drr_type);
1241                 dmu_ot_byteswap[byteswap].ob_func(data, drrw->drr_length);
1242         }
1243         dmu_write(os, drrw->drr_object,
1244             drrw->drr_offset, drrw->drr_length, data, tx);
1245         dmu_tx_commit(tx);
1246         return (0);
1247 }
1248
1249 /*
1250  * Handle a DRR_WRITE_BYREF record.  This record is used in dedup'ed
1251  * streams to refer to a copy of the data that is already on the
1252  * system because it came in earlier in the stream.  This function
1253  * finds the earlier copy of the data, and uses that copy instead of
1254  * data from the stream to fulfill this write.
1255  */
1256 static int
1257 restore_write_byref(struct restorearg *ra, objset_t *os,
1258     struct drr_write_byref *drrwbr)
1259 {
1260         dmu_tx_t *tx;
1261         int err;
1262         guid_map_entry_t gmesrch;
1263         guid_map_entry_t *gmep;
1264         avl_index_t     where;
1265         objset_t *ref_os = NULL;
1266         dmu_buf_t *dbp;
1267
1268         if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset)
1269                 return (EINVAL);
1270
1271         /*
1272          * If the GUID of the referenced dataset is different from the
1273          * GUID of the target dataset, find the referenced dataset.
1274          */
1275         if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
1276                 gmesrch.guid = drrwbr->drr_refguid;
1277                 if ((gmep = avl_find(ra->guid_to_ds_map, &gmesrch,
1278                     &where)) == NULL) {
1279                         return (EINVAL);
1280                 }
1281                 if (dmu_objset_from_ds(gmep->gme_ds, &ref_os))
1282                         return (EINVAL);
1283         } else {
1284                 ref_os = os;
1285         }
1286
1287         err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
1288             drrwbr->drr_refoffset, FTAG, &dbp, DMU_READ_PREFETCH);
1289         if (err)
1290                 return (err);
1291
1292         tx = dmu_tx_create(os);
1293
1294         dmu_tx_hold_write(tx, drrwbr->drr_object,
1295             drrwbr->drr_offset, drrwbr->drr_length);
1296         err = dmu_tx_assign(tx, TXG_WAIT);
1297         if (err) {
1298                 dmu_tx_abort(tx);
1299                 return (err);
1300         }
1301         dmu_write(os, drrwbr->drr_object,
1302             drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
1303         dmu_buf_rele(dbp, FTAG);
1304         dmu_tx_commit(tx);
1305         return (0);
1306 }
1307
1308 static int
1309 restore_spill(struct restorearg *ra, objset_t *os, struct drr_spill *drrs)
1310 {
1311         dmu_tx_t *tx;
1312         void *data;
1313         dmu_buf_t *db, *db_spill;
1314         int err;
1315
1316         if (drrs->drr_length < SPA_MINBLOCKSIZE ||
1317             drrs->drr_length > SPA_MAXBLOCKSIZE)
1318                 return (EINVAL);
1319
1320         data = restore_read(ra, drrs->drr_length);
1321         if (data == NULL)
1322                 return (ra->err);
1323
1324         if (dmu_object_info(os, drrs->drr_object, NULL) != 0)
1325                 return (EINVAL);
1326
1327         VERIFY(0 == dmu_bonus_hold(os, drrs->drr_object, FTAG, &db));
1328         if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
1329                 dmu_buf_rele(db, FTAG);
1330                 return (err);
1331         }
1332
1333         tx = dmu_tx_create(os);
1334
1335         dmu_tx_hold_spill(tx, db->db_object);
1336
1337         err = dmu_tx_assign(tx, TXG_WAIT);
1338         if (err) {
1339                 dmu_buf_rele(db, FTAG);
1340                 dmu_buf_rele(db_spill, FTAG);
1341                 dmu_tx_abort(tx);
1342                 return (err);
1343         }
1344         dmu_buf_will_dirty(db_spill, tx);
1345
1346         if (db_spill->db_size < drrs->drr_length)
1347                 VERIFY(0 == dbuf_spill_set_blksz(db_spill,
1348                     drrs->drr_length, tx));
1349         bcopy(data, db_spill->db_data, drrs->drr_length);
1350
1351         dmu_buf_rele(db, FTAG);
1352         dmu_buf_rele(db_spill, FTAG);
1353
1354         dmu_tx_commit(tx);
1355         return (0);
1356 }
1357
1358 /* ARGSUSED */
1359 noinline static int
1360 restore_free(struct restorearg *ra, objset_t *os,
1361     struct drr_free *drrf)
1362 {
1363         int err;
1364
1365         if (drrf->drr_length != -1ULL &&
1366             drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
1367                 return (EINVAL);
1368
1369         if (dmu_object_info(os, drrf->drr_object, NULL) != 0)
1370                 return (EINVAL);
1371
1372         err = dmu_free_long_range(os, drrf->drr_object,
1373             drrf->drr_offset, drrf->drr_length);
1374         return (err);
1375 }
1376
1377 /*
1378  * NB: callers *must* call dmu_recv_end() if this succeeds.
1379  */
1380 int
1381 dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
1382     int cleanup_fd, uint64_t *action_handlep)
1383 {
1384         struct restorearg ra = { 0 };
1385         dmu_replay_record_t *drr;
1386         objset_t *os;
1387         zio_cksum_t pcksum;
1388         int featureflags;
1389
1390         if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC))
1391                 ra.byteswap = TRUE;
1392
1393         {
1394                 /* compute checksum of drr_begin record */
1395                 dmu_replay_record_t *drr;
1396                 drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
1397
1398                 drr->drr_type = DRR_BEGIN;
1399                 drr->drr_u.drr_begin = *drc->drc_drrb;
1400                 if (ra.byteswap) {
1401                         fletcher_4_incremental_byteswap(drr,
1402                             sizeof (dmu_replay_record_t), &ra.cksum);
1403                 } else {
1404                         fletcher_4_incremental_native(drr,
1405                             sizeof (dmu_replay_record_t), &ra.cksum);
1406                 }
1407                 kmem_free(drr, sizeof (dmu_replay_record_t));
1408         }
1409
1410         if (ra.byteswap) {
1411                 struct drr_begin *drrb = drc->drc_drrb;
1412                 drrb->drr_magic = BSWAP_64(drrb->drr_magic);
1413                 drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
1414                 drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
1415                 drrb->drr_type = BSWAP_32(drrb->drr_type);
1416                 drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
1417                 drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
1418         }
1419
1420         ra.vp = vp;
1421         ra.voff = *voffp;
1422         ra.bufsize = 1<<20;
1423         ra.buf = vmem_alloc(ra.bufsize, KM_SLEEP);
1424
1425         /* these were verified in dmu_recv_begin */
1426         ASSERT(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo) ==
1427             DMU_SUBSTREAM);
1428         ASSERT(drc->drc_drrb->drr_type < DMU_OST_NUMTYPES);
1429
1430         /*
1431          * Open the objset we are modifying.
1432          */
1433         VERIFY(dmu_objset_from_ds(drc->drc_real_ds, &os) == 0);
1434
1435         ASSERT(drc->drc_real_ds->ds_phys->ds_flags & DS_FLAG_INCONSISTENT);
1436
1437         featureflags = DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);
1438
1439         /* if this stream is dedup'ed, set up the avl tree for guid mapping */
1440         if (featureflags & DMU_BACKUP_FEATURE_DEDUP) {
1441                 minor_t minor;
1442
1443                 if (cleanup_fd == -1) {
1444                         ra.err = EBADF;
1445                         goto out;
1446                 }
1447                 ra.err = zfs_onexit_fd_hold(cleanup_fd, &minor);
1448                 if (ra.err) {
1449                         cleanup_fd = -1;
1450                         goto out;
1451                 }
1452
1453                 if (*action_handlep == 0) {
1454                         ra.guid_to_ds_map =
1455                             kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
1456                         avl_create(ra.guid_to_ds_map, guid_compare,
1457                             sizeof (guid_map_entry_t),
1458                             offsetof(guid_map_entry_t, avlnode));
1459                         ra.err = zfs_onexit_add_cb(minor,
1460                             free_guid_map_onexit, ra.guid_to_ds_map,
1461                             action_handlep);
1462                         if (ra.err)
1463                                 goto out;
1464                 } else {
1465                         ra.err = zfs_onexit_cb_data(minor, *action_handlep,
1466                             (void **)&ra.guid_to_ds_map);
1467                         if (ra.err)
1468                                 goto out;
1469                 }
1470
1471                 drc->drc_guid_to_ds_map = ra.guid_to_ds_map;
1472         }
1473
1474         /*
1475          * Read records and process them.
1476          */
1477         pcksum = ra.cksum;
1478         while (ra.err == 0 &&
1479             NULL != (drr = restore_read(&ra, sizeof (*drr)))) {
1480                 if (issig(JUSTLOOKING) && issig(FORREAL)) {
1481                         ra.err = EINTR;
1482                         goto out;
1483                 }
1484
1485                 if (ra.byteswap)
1486                         backup_byteswap(drr);
1487
1488                 switch (drr->drr_type) {
1489                 case DRR_OBJECT:
1490                 {
1491                         /*
1492                          * We need to make a copy of the record header,
1493                          * because restore_{object,write} may need to
1494                          * restore_read(), which will invalidate drr.
1495                          */
1496                         struct drr_object drro = drr->drr_u.drr_object;
1497                         ra.err = restore_object(&ra, os, &drro);
1498                         break;
1499                 }
1500                 case DRR_FREEOBJECTS:
1501                 {
1502                         struct drr_freeobjects drrfo =
1503                             drr->drr_u.drr_freeobjects;
1504                         ra.err = restore_freeobjects(&ra, os, &drrfo);
1505                         break;
1506                 }
1507                 case DRR_WRITE:
1508                 {
1509                         struct drr_write drrw = drr->drr_u.drr_write;
1510                         ra.err = restore_write(&ra, os, &drrw);
1511                         break;
1512                 }
1513                 case DRR_WRITE_BYREF:
1514                 {
1515                         struct drr_write_byref drrwbr =
1516                             drr->drr_u.drr_write_byref;
1517                         ra.err = restore_write_byref(&ra, os, &drrwbr);
1518                         break;
1519                 }
1520                 case DRR_FREE:
1521                 {
1522                         struct drr_free drrf = drr->drr_u.drr_free;
1523                         ra.err = restore_free(&ra, os, &drrf);
1524                         break;
1525                 }
1526                 case DRR_END:
1527                 {
1528                         struct drr_end drre = drr->drr_u.drr_end;
1529                         /*
1530                          * We compare against the *previous* checksum
1531                          * value, because the stored checksum is of
1532                          * everything before the DRR_END record.
1533                          */
1534                         if (!ZIO_CHECKSUM_EQUAL(drre.drr_checksum, pcksum))
1535                                 ra.err = ECKSUM;
1536                         goto out;
1537                 }
1538                 case DRR_SPILL:
1539                 {
1540                         struct drr_spill drrs = drr->drr_u.drr_spill;
1541                         ra.err = restore_spill(&ra, os, &drrs);
1542                         break;
1543                 }
1544                 default:
1545                         ra.err = EINVAL;
1546                         goto out;
1547                 }
1548                 pcksum = ra.cksum;
1549         }
1550         ASSERT(ra.err != 0);
1551
1552 out:
1553         if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
1554                 zfs_onexit_fd_rele(cleanup_fd);
1555
1556         if (ra.err != 0) {
1557                 /*
1558                  * destroy what we created, so we don't leave it in the
1559                  * inconsistent restoring state.
1560                  */
1561                 txg_wait_synced(drc->drc_real_ds->ds_dir->dd_pool, 0);
1562
1563                 (void) dsl_dataset_destroy(drc->drc_real_ds, dmu_recv_tag,
1564                     B_FALSE);
1565                 if (drc->drc_real_ds != drc->drc_logical_ds) {
1566                         mutex_exit(&drc->drc_logical_ds->ds_recvlock);
1567                         dsl_dataset_rele(drc->drc_logical_ds, dmu_recv_tag);
1568                 }
1569         }
1570
1571         vmem_free(ra.buf, ra.bufsize);
1572         *voffp = ra.voff;
1573         return (ra.err);
1574 }
1575
1576 struct recvendsyncarg {
1577         char *tosnap;
1578         uint64_t creation_time;
1579         uint64_t toguid;
1580 };
1581
1582 static int
1583 recv_end_check(void *arg1, void *arg2, dmu_tx_t *tx)
1584 {
1585         dsl_dataset_t *ds = arg1;
1586         struct recvendsyncarg *resa = arg2;
1587
1588         return (dsl_dataset_snapshot_check(ds, resa->tosnap, tx));
1589 }
1590
1591 static void
1592 recv_end_sync(void *arg1, void *arg2, dmu_tx_t *tx)
1593 {
1594         dsl_dataset_t *ds = arg1;
1595         struct recvendsyncarg *resa = arg2;
1596
1597         dsl_dataset_snapshot_sync(ds, resa->tosnap, tx);
1598
1599         /* set snapshot's creation time and guid */
1600         dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
1601         ds->ds_prev->ds_phys->ds_creation_time = resa->creation_time;
1602         ds->ds_prev->ds_phys->ds_guid = resa->toguid;
1603         ds->ds_prev->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
1604
1605         dmu_buf_will_dirty(ds->ds_dbuf, tx);
1606         ds->ds_phys->ds_flags &= ~DS_FLAG_INCONSISTENT;
1607 }
1608
1609 static int
1610 add_ds_to_guidmap(avl_tree_t *guid_map, dsl_dataset_t *ds)
1611 {
1612         dsl_pool_t *dp = ds->ds_dir->dd_pool;
1613         uint64_t snapobj = ds->ds_phys->ds_prev_snap_obj;
1614         dsl_dataset_t *snapds;
1615         guid_map_entry_t *gmep;
1616         int err;
1617
1618         ASSERT(guid_map != NULL);
1619
1620         rw_enter(&dp->dp_config_rwlock, RW_READER);
1621         err = dsl_dataset_hold_obj(dp, snapobj, guid_map, &snapds);
1622         if (err == 0) {
1623                 gmep = kmem_alloc(sizeof (guid_map_entry_t), KM_SLEEP);
1624                 gmep->guid = snapds->ds_phys->ds_guid;
1625                 gmep->gme_ds = snapds;
1626                 avl_add(guid_map, gmep);
1627         }
1628
1629         rw_exit(&dp->dp_config_rwlock);
1630         return (err);
1631 }
1632
1633 static int
1634 dmu_recv_existing_end(dmu_recv_cookie_t *drc)
1635 {
1636         struct recvendsyncarg resa;
1637         dsl_dataset_t *ds = drc->drc_logical_ds;
1638         int err, myerr;
1639
1640         if (dsl_dataset_tryown(ds, FALSE, dmu_recv_tag)) {
1641                 err = dsl_dataset_clone_swap(drc->drc_real_ds, ds,
1642                     drc->drc_force);
1643                 if (err)
1644                         goto out;
1645         } else {
1646                 mutex_exit(&ds->ds_recvlock);
1647                 dsl_dataset_rele(ds, dmu_recv_tag);
1648                 (void) dsl_dataset_destroy(drc->drc_real_ds, dmu_recv_tag,
1649                     B_FALSE);
1650                 return (EBUSY);
1651         }
1652
1653         resa.creation_time = drc->drc_drrb->drr_creation_time;
1654         resa.toguid = drc->drc_drrb->drr_toguid;
1655         resa.tosnap = drc->drc_tosnap;
1656
1657         err = dsl_sync_task_do(ds->ds_dir->dd_pool,
1658             recv_end_check, recv_end_sync, ds, &resa, 3);
1659         if (err) {
1660                 /* swap back */
1661                 (void) dsl_dataset_clone_swap(drc->drc_real_ds, ds, B_TRUE);
1662         }
1663
1664 out:
1665         mutex_exit(&ds->ds_recvlock);
1666         if (err == 0 && drc->drc_guid_to_ds_map != NULL)
1667                 (void) add_ds_to_guidmap(drc->drc_guid_to_ds_map, ds);
1668         dsl_dataset_disown(ds, dmu_recv_tag);
1669         myerr = dsl_dataset_destroy(drc->drc_real_ds, dmu_recv_tag, B_FALSE);
1670         ASSERT0(myerr);
1671         return (err);
1672 }
1673
1674 static int
1675 dmu_recv_new_end(dmu_recv_cookie_t *drc)
1676 {
1677         struct recvendsyncarg resa;
1678         dsl_dataset_t *ds = drc->drc_logical_ds;
1679         int err;
1680
1681         /*
1682          * XXX hack; seems the ds is still dirty and dsl_pool_zil_clean()
1683          * expects it to have a ds_user_ptr (and zil), but clone_swap()
1684          * can close it.
1685          */
1686         txg_wait_synced(ds->ds_dir->dd_pool, 0);
1687
1688         resa.creation_time = drc->drc_drrb->drr_creation_time;
1689         resa.toguid = drc->drc_drrb->drr_toguid;
1690         resa.tosnap = drc->drc_tosnap;
1691
1692         err = dsl_sync_task_do(ds->ds_dir->dd_pool,
1693             recv_end_check, recv_end_sync, ds, &resa, 3);
1694         if (err) {
1695                 /* clean up the fs we just recv'd into */
1696                 (void) dsl_dataset_destroy(ds, dmu_recv_tag, B_FALSE);
1697         } else {
1698                 if (drc->drc_guid_to_ds_map != NULL)
1699                         (void) add_ds_to_guidmap(drc->drc_guid_to_ds_map, ds);
1700                 /* release the hold from dmu_recv_begin */
1701                 dsl_dataset_disown(ds, dmu_recv_tag);
1702         }
1703         return (err);
1704 }
1705
1706 int
1707 dmu_recv_end(dmu_recv_cookie_t *drc)
1708 {
1709         if (drc->drc_logical_ds != drc->drc_real_ds)
1710                 return (dmu_recv_existing_end(drc));
1711         else
1712                 return (dmu_recv_new_end(drc));
1713 }