zfs/lib/libzpool/vdev.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #pragma ident   "@(#)vdev.c     1.33    07/11/27 SMI"
  28
  29 #include <sys/zfs_context.h>
  30 #include <sys/fm/fs/zfs.h>
  31 #include <sys/spa.h>
  32 #include <sys/spa_impl.h>
  33 #include <sys/dmu.h>
  34 #include <sys/dmu_tx.h>
  35 #include <sys/vdev_impl.h>
  36 #include <sys/uberblock_impl.h>
  37 #include <sys/metaslab.h>
  38 #include <sys/metaslab_impl.h>
  39 #include <sys/space_map.h>
  40 #include <sys/zio.h>
  41 #include <sys/zap.h>
  42 #include <sys/fs/zfs.h>
  43
  44 /*
  45  * Virtual device management.
  46  */
  47
  48 static vdev_ops_t *vdev_ops_table[] = {
  49         &vdev_root_ops,
  50         &vdev_raidz_ops,
  51         &vdev_mirror_ops,
  52         &vdev_replacing_ops,
  53         &vdev_spare_ops,
  54         &vdev_disk_ops,
  55         &vdev_file_ops,
  56         &vdev_missing_ops,
  57         NULL
  58 };
  59
  60 /* maximum scrub/resilver I/O queue */
  61 int zfs_scrub_limit = 70;
  62
  63 /*
  64  * Given a vdev type, return the appropriate ops vector.
  65  */
  66 static vdev_ops_t *
  67 vdev_getops(const char *type)
  68 {
  69         vdev_ops_t *ops, **opspp;
  70
  71         for (opspp = vdev_ops_table; (ops = *opspp) != NULL; opspp++)
  72                 if (strcmp(ops->vdev_op_type, type) == 0)
  73                         break;
  74
  75         return (ops);
  76 }
  77
  78 /*
  79  * Default asize function: return the MAX of psize with the asize of
  80  * all children.  This is what's used by anything other than RAID-Z.
  81  */
  82 uint64_t
  83 vdev_default_asize(vdev_t *vd, uint64_t psize)
  84 {
  85         uint64_t asize = P2ROUNDUP(psize, 1ULL << vd->vdev_top->vdev_ashift);
  86         uint64_t csize;
  87         uint64_t c;
  88
  89         for (c = 0; c < vd->vdev_children; c++) {
  90                 csize = vdev_psize_to_asize(vd->vdev_child[c], psize);
  91                 asize = MAX(asize, csize);
  92         }
  93
  94         return (asize);
  95 }
  96
  97 /*
  98  * Get the replaceable or attachable device size.
  99  * If the parent is a mirror or raidz, the replaceable size is the minimum
 100  * psize of all its children. For the rest, just return our own psize.
 101  *
 102  * e.g.
 103  *                      psize   rsize
 104  * root                 -       -
 105  *      mirror/raidz    -       -
 106  *          disk1       20g     20g
 107  *          disk2       40g     20g
 108  *      disk3           80g     80g
 109  */
 110 uint64_t
 111 vdev_get_rsize(vdev_t *vd)
 112 {
 113         vdev_t *pvd, *cvd;
 114         uint64_t c, rsize;
 115
 116         pvd = vd->vdev_parent;
 117
 118         /*
 119          * If our parent is NULL or the root, just return our own psize.
 120          */
 121         if (pvd == NULL || pvd->vdev_parent == NULL)
 122                 return (vd->vdev_psize);
 123
 124         rsize = 0;
 125
 126         for (c = 0; c < pvd->vdev_children; c++) {
 127                 cvd = pvd->vdev_child[c];
 128                 rsize = MIN(rsize - 1, cvd->vdev_psize - 1) + 1;
 129         }
 130
 131         return (rsize);
 132 }
 133
 134 vdev_t *
 135 vdev_lookup_top(spa_t *spa, uint64_t vdev)
 136 {
 137         vdev_t *rvd = spa->spa_root_vdev;
 138
 139         ASSERT(spa_config_held(spa, RW_READER) ||
 140             curthread == spa->spa_scrub_thread);
 141
 142         if (vdev < rvd->vdev_children)
 143                 return (rvd->vdev_child[vdev]);
 144
 145         return (NULL);
 146 }
 147
 148 vdev_t *
 149 vdev_lookup_by_guid(vdev_t *vd, uint64_t guid)
 150 {
 151         int c;
 152         vdev_t *mvd;
 153
 154         if (vd->vdev_guid == guid)
 155                 return (vd);
 156
 157         for (c = 0; c < vd->vdev_children; c++)
 158                 if ((mvd = vdev_lookup_by_guid(vd->vdev_child[c], guid)) !=
 159                     NULL)
 160                         return (mvd);
 161
 162         return (NULL);
 163 }
 164
 165 void
 166 vdev_add_child(vdev_t *pvd, vdev_t *cvd)
 167 {
 168         size_t oldsize, newsize;
 169         uint64_t id = cvd->vdev_id;
 170         vdev_t **newchild;
 171
 172         ASSERT(spa_config_held(cvd->vdev_spa, RW_WRITER));
 173         ASSERT(cvd->vdev_parent == NULL);
 174
 175         cvd->vdev_parent = pvd;
 176
 177         if (pvd == NULL)
 178                 return;
 179
 180         ASSERT(id >= pvd->vdev_children || pvd->vdev_child[id] == NULL);
 181
 182         oldsize = pvd->vdev_children * sizeof (vdev_t *);
 183         pvd->vdev_children = MAX(pvd->vdev_children, id + 1);
 184         newsize = pvd->vdev_children * sizeof (vdev_t *);
 185
 186         newchild = kmem_zalloc(newsize, KM_SLEEP);
 187         if (pvd->vdev_child != NULL) {
 188                 bcopy(pvd->vdev_child, newchild, oldsize);
 189                 kmem_free(pvd->vdev_child, oldsize);
 190         }
 191
 192         pvd->vdev_child = newchild;
 193         pvd->vdev_child[id] = cvd;
 194
 195         cvd->vdev_top = (pvd->vdev_top ? pvd->vdev_top: cvd);
 196         ASSERT(cvd->vdev_top->vdev_parent->vdev_parent == NULL);
 197
 198         /*
 199          * Walk up all ancestors to update guid sum.
 200          */
 201         for (; pvd != NULL; pvd = pvd->vdev_parent)
 202                 pvd->vdev_guid_sum += cvd->vdev_guid_sum;
 203
 204         if (cvd->vdev_ops->vdev_op_leaf)
 205                 cvd->vdev_spa->spa_scrub_maxinflight += zfs_scrub_limit;
 206 }
 207
 208 void
 209 vdev_remove_child(vdev_t *pvd, vdev_t *cvd)
 210 {
 211         int c;
 212         uint_t id = cvd->vdev_id;
 213
 214         ASSERT(cvd->vdev_parent == pvd);
 215
 216         if (pvd == NULL)
 217                 return;
 218
 219         ASSERT(id < pvd->vdev_children);
 220         ASSERT(pvd->vdev_child[id] == cvd);
 221
 222         pvd->vdev_child[id] = NULL;
 223         cvd->vdev_parent = NULL;
 224
 225         for (c = 0; c < pvd->vdev_children; c++)
 226                 if (pvd->vdev_child[c])
 227                         break;
 228
 229         if (c == pvd->vdev_children) {
 230                 kmem_free(pvd->vdev_child, c * sizeof (vdev_t *));
 231                 pvd->vdev_child = NULL;
 232                 pvd->vdev_children = 0;
 233         }
 234
 235         /*
 236          * Walk up all ancestors to update guid sum.
 237          */
 238         for (; pvd != NULL; pvd = pvd->vdev_parent)
 239                 pvd->vdev_guid_sum -= cvd->vdev_guid_sum;
 240
 241         if (cvd->vdev_ops->vdev_op_leaf)
 242                 cvd->vdev_spa->spa_scrub_maxinflight -= zfs_scrub_limit;
 243 }
 244
 245 /*
 246  * Remove any holes in the child array.
 247  */
 248 void
 249 vdev_compact_children(vdev_t *pvd)
 250 {
 251         vdev_t **newchild, *cvd;
 252         int oldc = pvd->vdev_children;
 253         int newc, c;
 254
 255         ASSERT(spa_config_held(pvd->vdev_spa, RW_WRITER));
 256
 257         for (c = newc = 0; c < oldc; c++)
 258                 if (pvd->vdev_child[c])
 259                         newc++;
 260
 261         newchild = kmem_alloc(newc * sizeof (vdev_t *), KM_SLEEP);
 262
 263         for (c = newc = 0; c < oldc; c++) {
 264                 if ((cvd = pvd->vdev_child[c]) != NULL) {
 265                         newchild[newc] = cvd;
 266                         cvd->vdev_id = newc++;
 267                 }
 268         }
 269
 270         kmem_free(pvd->vdev_child, oldc * sizeof (vdev_t *));
 271         pvd->vdev_child = newchild;
 272         pvd->vdev_children = newc;
 273 }
 274
 275 /*
 276  * Allocate and minimally initialize a vdev_t.
 277  */
 278 static vdev_t *
 279 vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops)
 280 {
 281         vdev_t *vd;
 282
 283         vd = kmem_zalloc(sizeof (vdev_t), KM_SLEEP);
 284
 285         if (spa->spa_root_vdev == NULL) {
 286                 ASSERT(ops == &vdev_root_ops);
 287                 spa->spa_root_vdev = vd;
 288         }
 289
 290         if (guid == 0) {
 291                 if (spa->spa_root_vdev == vd) {
 292                         /*
 293                          * The root vdev's guid will also be the pool guid,
 294                          * which must be unique among all pools.
 295                          */
 296                         while (guid == 0 || spa_guid_exists(guid, 0))
 297                                 guid = spa_get_random(-1ULL);
 298                 } else {
 299                         /*
 300                          * Any other vdev's guid must be unique within the pool.
 301                          */
 302                         while (guid == 0 ||
 303                             spa_guid_exists(spa_guid(spa), guid))
 304                                 guid = spa_get_random(-1ULL);
 305                 }
 306                 ASSERT(!spa_guid_exists(spa_guid(spa), guid));
 307         }
 308
 309         vd->vdev_spa = spa;
 310         vd->vdev_id = id;
 311         vd->vdev_guid = guid;
 312         vd->vdev_guid_sum = guid;
 313         vd->vdev_ops = ops;
 314         vd->vdev_state = VDEV_STATE_CLOSED;
 315
 316         mutex_init(&vd->vdev_dtl_lock, NULL, MUTEX_DEFAULT, NULL);
 317         mutex_init(&vd->vdev_stat_lock, NULL, MUTEX_DEFAULT, NULL);
 318         space_map_create(&vd->vdev_dtl_map, 0, -1ULL, 0, &vd->vdev_dtl_lock);
 319         space_map_create(&vd->vdev_dtl_scrub, 0, -1ULL, 0, &vd->vdev_dtl_lock);
 320         txg_list_create(&vd->vdev_ms_list,
 321             offsetof(struct metaslab, ms_txg_node));
 322         txg_list_create(&vd->vdev_dtl_list,
 323             offsetof(struct vdev, vdev_dtl_node));
 324         vd->vdev_stat.vs_timestamp = gethrtime();
 325         vdev_queue_init(vd);
 326         vdev_cache_init(vd);
 327
 328         return (vd);
 329 }
 330
 331 /*
 332  * Allocate a new vdev.  The 'alloctype' is used to control whether we are
 333  * creating a new vdev or loading an existing one - the behavior is slightly
 334  * different for each case.
 335  */
 336 int
 337 vdev_alloc(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id,
 338     int alloctype)
 339 {
 340         vdev_ops_t *ops;
 341         char *type;
 342         uint64_t guid = 0, islog, nparity;
 343         vdev_t *vd;
 344
 345         ASSERT(spa_config_held(spa, RW_WRITER));
 346
 347         if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0)
 348                 return (EINVAL);
 349
 350         if ((ops = vdev_getops(type)) == NULL)
 351                 return (EINVAL);
 352
 353         /*
 354          * If this is a load, get the vdev guid from the nvlist.
 355          * Otherwise, vdev_alloc_common() will generate one for us.
 356          */
 357         if (alloctype == VDEV_ALLOC_LOAD) {
 358                 uint64_t label_id;
 359
 360                 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ID, &label_id) ||
 361                     label_id != id)
 362                         return (EINVAL);
 363
 364                 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
 365                         return (EINVAL);
 366         } else if (alloctype == VDEV_ALLOC_SPARE) {
 367                 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
 368                         return (EINVAL);
 369         } else if (alloctype == VDEV_ALLOC_L2CACHE) {
 370                 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
 371                         return (EINVAL);
 372         }
 373
 374         /*
 375          * The first allocated vdev must be of type 'root'.
 376          */
 377         if (ops != &vdev_root_ops && spa->spa_root_vdev == NULL)
 378                 return (EINVAL);
 379
 380         /*
 381          * Determine whether we're a log vdev.
 382          */
 383         islog = 0;
 384         (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG, &islog);
 385         if (islog && spa_version(spa) < SPA_VERSION_SLOGS)
 386                 return (ENOTSUP);
 387
 388         /*
 389          * Set the nparity property for RAID-Z vdevs.
 390          */
 391         nparity = -1ULL;
 392         if (ops == &vdev_raidz_ops) {
 393                 if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NPARITY,
 394                     &nparity) == 0) {
 395                         /*
 396                          * Currently, we can only support 2 parity devices.
 397                          */
 398                         if (nparity == 0 || nparity > 2)
 399                                 return (EINVAL);
 400                         /*
 401                          * Older versions can only support 1 parity device.
 402                          */
 403                         if (nparity == 2 &&
 404                             spa_version(spa) < SPA_VERSION_RAID6)
 405                                 return (ENOTSUP);
 406                 } else {
 407                         /*
 408                          * We require the parity to be specified for SPAs that
 409                          * support multiple parity levels.
 410                          */
 411                         if (spa_version(spa) >= SPA_VERSION_RAID6)
 412                                 return (EINVAL);
 413                         /*
 414                          * Otherwise, we default to 1 parity device for RAID-Z.
 415                          */
 416                         nparity = 1;
 417                 }
 418         } else {
 419                 nparity = 0;
 420         }
 421         ASSERT(nparity != -1ULL);
 422
 423         vd = vdev_alloc_common(spa, id, guid, ops);
 424
 425         vd->vdev_islog = islog;
 426         vd->vdev_nparity = nparity;
 427
 428         if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &vd->vdev_path) == 0)
 429                 vd->vdev_path = spa_strdup(vd->vdev_path);
 430         if (nvlist_lookup_string(nv, ZPOOL_CONFIG_DEVID, &vd->vdev_devid) == 0)
 431                 vd->vdev_devid = spa_strdup(vd->vdev_devid);
 432         if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PHYS_PATH,
 433             &vd->vdev_physpath) == 0)
 434                 vd->vdev_physpath = spa_strdup(vd->vdev_physpath);
 435
 436         /*
 437          * Set the whole_disk property.  If it's not specified, leave the value
 438          * as -1.
 439          */
 440         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_WHOLE_DISK,
 441             &vd->vdev_wholedisk) != 0)
 442                 vd->vdev_wholedisk = -1ULL;
 443
 444         /*
 445          * Look for the 'not present' flag.  This will only be set if the device
 446          * was not present at the time of import.
 447          */
 448         (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
 449             &vd->vdev_not_present);
 450
 451         /*
 452          * Get the alignment requirement.
 453          */
 454         (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASHIFT, &vd->vdev_ashift);
 455
 456         /*
 457          * If we're a top-level vdev, try to load the allocation parameters.
 458          */
 459         if (parent && !parent->vdev_parent && alloctype == VDEV_ALLOC_LOAD) {
 460                 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_ARRAY,
 461                     &vd->vdev_ms_array);
 462                 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_METASLAB_SHIFT,
 463                     &vd->vdev_ms_shift);
 464                 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_ASIZE,
 465                     &vd->vdev_asize);
 466         }
 467
 468         /*
 469          * If we're a leaf vdev, try to load the DTL object and other state.
 470          */
 471         if (vd->vdev_ops->vdev_op_leaf && alloctype == VDEV_ALLOC_LOAD) {
 472                 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DTL,
 473                     &vd->vdev_dtl.smo_object);
 474                 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_OFFLINE,
 475                     &vd->vdev_offline);
 476                 (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_UNSPARE,
 477                     &vd->vdev_unspare);
 478                 /*
 479                  * When importing a pool, we want to ignore the persistent fault
 480                  * state, as the diagnosis made on another system may not be
 481                  * valid in the current context.
 482                  */
 483                 if (spa->spa_load_state == SPA_LOAD_OPEN) {
 484                         (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_FAULTED,
 485                             &vd->vdev_faulted);
 486                         (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_DEGRADED,
 487                             &vd->vdev_degraded);
 488                         (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_REMOVED,
 489                             &vd->vdev_removed);
 490                 }
 491         }
 492
 493         /*
 494          * Add ourselves to the parent's list of children.
 495          */
 496         vdev_add_child(parent, vd);
 497
 498         *vdp = vd;
 499
 500         return (0);
 501 }
 502
 503 void
 504 vdev_free(vdev_t *vd)
 505 {
 506         int c;
 507         spa_t *spa = vd->vdev_spa;
 508
 509         /*
 510          * vdev_free() implies closing the vdev first.  This is simpler than
 511          * trying to ensure complicated semantics for all callers.
 512          */
 513         vdev_close(vd);
 514
 515
 516         ASSERT(!list_link_active(&vd->vdev_dirty_node));
 517
 518         /*
 519          * Free all children.
 520          */
 521         for (c = 0; c < vd->vdev_children; c++)
 522                 vdev_free(vd->vdev_child[c]);
 523
 524         ASSERT(vd->vdev_child == NULL);
 525         ASSERT(vd->vdev_guid_sum == vd->vdev_guid);
 526
 527         /*
 528          * Discard allocation state.
 529          */
 530         if (vd == vd->vdev_top)
 531                 vdev_metaslab_fini(vd);
 532
 533         ASSERT3U(vd->vdev_stat.vs_space, ==, 0);
 534         ASSERT3U(vd->vdev_stat.vs_dspace, ==, 0);
 535         ASSERT3U(vd->vdev_stat.vs_alloc, ==, 0);
 536
 537         /*
 538          * Remove this vdev from its parent's child list.
 539          */
 540         vdev_remove_child(vd->vdev_parent, vd);
 541
 542         ASSERT(vd->vdev_parent == NULL);
 543
 544         /*
 545          * Clean up vdev structure.
 546          */
 547         vdev_queue_fini(vd);
 548         vdev_cache_fini(vd);
 549
 550         if (vd->vdev_path)
 551                 spa_strfree(vd->vdev_path);
 552         if (vd->vdev_devid)
 553                 spa_strfree(vd->vdev_devid);
 554         if (vd->vdev_physpath)
 555                 spa_strfree(vd->vdev_physpath);
 556
 557         if (vd->vdev_isspare)
 558                 spa_spare_remove(vd);
 559         if (vd->vdev_isl2cache)
 560                 spa_l2cache_remove(vd);
 561
 562         txg_list_destroy(&vd->vdev_ms_list);
 563         txg_list_destroy(&vd->vdev_dtl_list);
 564         mutex_enter(&vd->vdev_dtl_lock);
 565         space_map_unload(&vd->vdev_dtl_map);
 566         space_map_destroy(&vd->vdev_dtl_map);
 567         space_map_vacate(&vd->vdev_dtl_scrub, NULL, NULL);
 568         space_map_destroy(&vd->vdev_dtl_scrub);
 569         mutex_exit(&vd->vdev_dtl_lock);
 570         mutex_destroy(&vd->vdev_dtl_lock);
 571         mutex_destroy(&vd->vdev_stat_lock);
 572
 573         if (vd == spa->spa_root_vdev)
 574                 spa->spa_root_vdev = NULL;
 575
 576         kmem_free(vd, sizeof (vdev_t));
 577 }
 578
 579 /*
 580  * Transfer top-level vdev state from svd to tvd.
 581  */
 582 static void
 583 vdev_top_transfer(vdev_t *svd, vdev_t *tvd)
 584 {
 585         spa_t *spa = svd->vdev_spa;
 586         metaslab_t *msp;
 587         vdev_t *vd;
 588         int t;
 589
 590         ASSERT(tvd == tvd->vdev_top);
 591
 592         tvd->vdev_ms_array = svd->vdev_ms_array;
 593         tvd->vdev_ms_shift = svd->vdev_ms_shift;
 594         tvd->vdev_ms_count = svd->vdev_ms_count;
 595
 596         svd->vdev_ms_array = 0;
 597         svd->vdev_ms_shift = 0;
 598         svd->vdev_ms_count = 0;
 599
 600         tvd->vdev_mg = svd->vdev_mg;
 601         tvd->vdev_ms = svd->vdev_ms;
 602
 603         svd->vdev_mg = NULL;
 604         svd->vdev_ms = NULL;
 605
 606         if (tvd->vdev_mg != NULL)
 607                 tvd->vdev_mg->mg_vd = tvd;
 608
 609         tvd->vdev_stat.vs_alloc = svd->vdev_stat.vs_alloc;
 610         tvd->vdev_stat.vs_space = svd->vdev_stat.vs_space;
 611         tvd->vdev_stat.vs_dspace = svd->vdev_stat.vs_dspace;
 612
 613         svd->vdev_stat.vs_alloc = 0;
 614         svd->vdev_stat.vs_space = 0;
 615         svd->vdev_stat.vs_dspace = 0;
 616
 617         for (t = 0; t < TXG_SIZE; t++) {
 618                 while ((msp = txg_list_remove(&svd->vdev_ms_list, t)) != NULL)
 619                         (void) txg_list_add(&tvd->vdev_ms_list, msp, t);
 620                 while ((vd = txg_list_remove(&svd->vdev_dtl_list, t)) != NULL)
 621                         (void) txg_list_add(&tvd->vdev_dtl_list, vd, t);
 622                 if (txg_list_remove_this(&spa->spa_vdev_txg_list, svd, t))
 623                         (void) txg_list_add(&spa->spa_vdev_txg_list, tvd, t);
 624         }
 625
 626         if (list_link_active(&svd->vdev_dirty_node)) {
 627                 vdev_config_clean(svd);
 628                 vdev_config_dirty(tvd);
 629         }
 630
 631         tvd->vdev_deflate_ratio = svd->vdev_deflate_ratio;
 632         svd->vdev_deflate_ratio = 0;
 633
 634         tvd->vdev_islog = svd->vdev_islog;
 635         svd->vdev_islog = 0;
 636 }
 637
 638 static void
 639 vdev_top_update(vdev_t *tvd, vdev_t *vd)
 640 {
 641         int c;
 642
 643         if (vd == NULL)
 644                 return;
 645
 646         vd->vdev_top = tvd;
 647
 648         for (c = 0; c < vd->vdev_children; c++)
 649                 vdev_top_update(tvd, vd->vdev_child[c]);
 650 }
 651
 652 /*
 653  * Add a mirror/replacing vdev above an existing vdev.
 654  */
 655 vdev_t *
 656 vdev_add_parent(vdev_t *cvd, vdev_ops_t *ops)
 657 {
 658         spa_t *spa = cvd->vdev_spa;
 659         vdev_t *pvd = cvd->vdev_parent;
 660         vdev_t *mvd;
 661
 662         ASSERT(spa_config_held(spa, RW_WRITER));
 663
 664         mvd = vdev_alloc_common(spa, cvd->vdev_id, 0, ops);
 665
 666         mvd->vdev_asize = cvd->vdev_asize;
 667         mvd->vdev_ashift = cvd->vdev_ashift;
 668         mvd->vdev_state = cvd->vdev_state;
 669
 670         vdev_remove_child(pvd, cvd);
 671         vdev_add_child(pvd, mvd);
 672         cvd->vdev_id = mvd->vdev_children;
 673         vdev_add_child(mvd, cvd);
 674         vdev_top_update(cvd->vdev_top, cvd->vdev_top);
 675
 676         if (mvd == mvd->vdev_top)
 677                 vdev_top_transfer(cvd, mvd);
 678
 679         return (mvd);
 680 }
 681
 682 /*
 683  * Remove a 1-way mirror/replacing vdev from the tree.
 684  */
 685 void
 686 vdev_remove_parent(vdev_t *cvd)
 687 {
 688         vdev_t *mvd = cvd->vdev_parent;
 689         vdev_t *pvd = mvd->vdev_parent;
 690
 691         ASSERT(spa_config_held(cvd->vdev_spa, RW_WRITER));
 692
 693         ASSERT(mvd->vdev_children == 1);
 694         ASSERT(mvd->vdev_ops == &vdev_mirror_ops ||
 695             mvd->vdev_ops == &vdev_replacing_ops ||
 696             mvd->vdev_ops == &vdev_spare_ops);
 697         cvd->vdev_ashift = mvd->vdev_ashift;
 698
 699         vdev_remove_child(mvd, cvd);
 700         vdev_remove_child(pvd, mvd);
 701         cvd->vdev_id = mvd->vdev_id;
 702         vdev_add_child(pvd, cvd);
 703         /*
 704          * If we created a new toplevel vdev, then we need to change the child's
 705          * vdev GUID to match the old toplevel vdev.  Otherwise, we could have
 706          * detached an offline device, and when we go to import the pool we'll
 707          * think we have two toplevel vdevs, instead of a different version of
 708          * the same toplevel vdev.
 709          */
 710         if (cvd->vdev_top == cvd) {
 711                 pvd->vdev_guid_sum -= cvd->vdev_guid;
 712                 cvd->vdev_guid_sum -= cvd->vdev_guid;
 713                 cvd->vdev_guid = mvd->vdev_guid;
 714                 cvd->vdev_guid_sum += mvd->vdev_guid;
 715                 pvd->vdev_guid_sum += cvd->vdev_guid;
 716         }
 717         vdev_top_update(cvd->vdev_top, cvd->vdev_top);
 718
 719         if (cvd == cvd->vdev_top)
 720                 vdev_top_transfer(mvd, cvd);
 721
 722         ASSERT(mvd->vdev_children == 0);
 723         vdev_free(mvd);
 724 }
 725
 726 int
 727 vdev_metaslab_init(vdev_t *vd, uint64_t txg)
 728 {
 729         spa_t *spa = vd->vdev_spa;
 730         objset_t *mos = spa->spa_meta_objset;
 731         metaslab_class_t *mc;
 732         uint64_t m;
 733         uint64_t oldc = vd->vdev_ms_count;
 734         uint64_t newc = vd->vdev_asize >> vd->vdev_ms_shift;
 735         metaslab_t **mspp;
 736         int error;
 737
 738         if (vd->vdev_ms_shift == 0)     /* not being allocated from yet */
 739                 return (0);
 740
 741         dprintf("%s oldc %llu newc %llu\n", vdev_description(vd), oldc, newc);
 742
 743         ASSERT(oldc <= newc);
 744
 745         if (vd->vdev_islog)
 746                 mc = spa->spa_log_class;
 747         else
 748                 mc = spa->spa_normal_class;
 749
 750         if (vd->vdev_mg == NULL)
 751                 vd->vdev_mg = metaslab_group_create(mc, vd);
 752
 753         mspp = kmem_zalloc(newc * sizeof (*mspp), KM_SLEEP);
 754
 755         if (oldc != 0) {
 756                 bcopy(vd->vdev_ms, mspp, oldc * sizeof (*mspp));
 757                 kmem_free(vd->vdev_ms, oldc * sizeof (*mspp));
 758         }
 759
 760         vd->vdev_ms = mspp;
 761         vd->vdev_ms_count = newc;
 762
 763         for (m = oldc; m < newc; m++) {
 764                 space_map_obj_t smo = { 0, 0, 0 };
 765                 if (txg == 0) {
 766                         uint64_t object = 0;
 767                         error = dmu_read(mos, vd->vdev_ms_array,
 768                             m * sizeof (uint64_t), sizeof (uint64_t), &object);
 769                         if (error)
 770                                 return (error);
 771                         if (object != 0) {
 772                                 dmu_buf_t *db;
 773                                 error = dmu_bonus_hold(mos, object, FTAG, &db);
 774                                 if (error)
 775                                         return (error);
 776                                 ASSERT3U(db->db_size, >=, sizeof (smo));
 777                                 bcopy(db->db_data, &smo, sizeof (smo));
 778                                 ASSERT3U(smo.smo_object, ==, object);
 779                                 dmu_buf_rele(db, FTAG);
 780                         }
 781                 }
 782                 vd->vdev_ms[m] = metaslab_init(vd->vdev_mg, &smo,
 783                     m << vd->vdev_ms_shift, 1ULL << vd->vdev_ms_shift, txg);
 784         }
 785
 786         return (0);
 787 }
 788
 789 void
 790 vdev_metaslab_fini(vdev_t *vd)
 791 {
 792         uint64_t m;
 793         uint64_t count = vd->vdev_ms_count;
 794
 795         if (vd->vdev_ms != NULL) {
 796                 for (m = 0; m < count; m++)
 797                         if (vd->vdev_ms[m] != NULL)
 798                                 metaslab_fini(vd->vdev_ms[m]);
 799                 kmem_free(vd->vdev_ms, count * sizeof (metaslab_t *));
 800                 vd->vdev_ms = NULL;
 801         }
 802 }
 803
 804 int
 805 vdev_probe(vdev_t *vd)
 806 {
 807         if (vd == NULL)
 808                 return (EINVAL);
 809
 810         /*
 811          * Right now we only support status checks on the leaf vdevs.
 812          */
 813         if (vd->vdev_ops->vdev_op_leaf)
 814                 return (vd->vdev_ops->vdev_op_probe(vd));
 815
 816         return (0);
 817 }
 818
 819 /*
 820  * Prepare a virtual device for access.
 821  */
 822 int
 823 vdev_open(vdev_t *vd)
 824 {
 825         int error;
 826         int c;
 827         uint64_t osize = 0;
 828         uint64_t asize, psize;
 829         uint64_t ashift = 0;
 830
 831         ASSERT(vd->vdev_state == VDEV_STATE_CLOSED ||
 832             vd->vdev_state == VDEV_STATE_CANT_OPEN ||
 833             vd->vdev_state == VDEV_STATE_OFFLINE);
 834
 835         if (vd->vdev_fault_mode == VDEV_FAULT_COUNT)
 836                 vd->vdev_fault_arg >>= 1;
 837         else
 838                 vd->vdev_fault_mode = VDEV_FAULT_NONE;
 839
 840         vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
 841
 842         if (!vd->vdev_removed && vd->vdev_faulted) {
 843                 ASSERT(vd->vdev_children == 0);
 844                 vdev_set_state(vd, B_TRUE, VDEV_STATE_FAULTED,
 845                     VDEV_AUX_ERR_EXCEEDED);
 846                 return (ENXIO);
 847         } else if (vd->vdev_offline) {
 848                 ASSERT(vd->vdev_children == 0);
 849                 vdev_set_state(vd, B_TRUE, VDEV_STATE_OFFLINE, VDEV_AUX_NONE);
 850                 return (ENXIO);
 851         }
 852
 853         error = vd->vdev_ops->vdev_op_open(vd, &osize, &ashift);
 854
 855         if (zio_injection_enabled && error == 0)
 856                 error = zio_handle_device_injection(vd, ENXIO);
 857
 858         if (error) {
 859                 if (vd->vdev_removed &&
 860                     vd->vdev_stat.vs_aux != VDEV_AUX_OPEN_FAILED)
 861                         vd->vdev_removed = B_FALSE;
 862
 863                 vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
 864                     vd->vdev_stat.vs_aux);
 865                 return (error);
 866         }
 867
 868         vd->vdev_removed = B_FALSE;
 869
 870         if (vd->vdev_degraded) {
 871                 ASSERT(vd->vdev_children == 0);
 872                 vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
 873                     VDEV_AUX_ERR_EXCEEDED);
 874         } else {
 875                 vd->vdev_state = VDEV_STATE_HEALTHY;
 876         }
 877
 878         for (c = 0; c < vd->vdev_children; c++)
 879                 if (vd->vdev_child[c]->vdev_state != VDEV_STATE_HEALTHY) {
 880                         vdev_set_state(vd, B_TRUE, VDEV_STATE_DEGRADED,
 881                             VDEV_AUX_NONE);
 882                         break;
 883                 }
 884
 885         osize = P2ALIGN(osize, (uint64_t)sizeof (vdev_label_t));
 886
 887         if (vd->vdev_children == 0) {
 888                 if (osize < SPA_MINDEVSIZE) {
 889                         vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
 890                             VDEV_AUX_TOO_SMALL);
 891                         return (EOVERFLOW);
 892                 }
 893                 psize = osize;
 894                 asize = osize - (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE);
 895         } else {
 896                 if (vd->vdev_parent != NULL && osize < SPA_MINDEVSIZE -
 897                     (VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE)) {
 898                         vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
 899                             VDEV_AUX_TOO_SMALL);
 900                         return (EOVERFLOW);
 901                 }
 902                 psize = 0;
 903                 asize = osize;
 904         }
 905
 906         vd->vdev_psize = psize;
 907
 908         if (vd->vdev_asize == 0) {
 909                 /*
 910                  * This is the first-ever open, so use the computed values.
 911                  * For testing purposes, a higher ashift can be requested.
 912                  */
 913                 vd->vdev_asize = asize;
 914                 vd->vdev_ashift = MAX(ashift, vd->vdev_ashift);
 915         } else {
 916                 /*
 917                  * Make sure the alignment requirement hasn't increased.
 918                  */
 919                 if (ashift > vd->vdev_top->vdev_ashift) {
 920                         vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
 921                             VDEV_AUX_BAD_LABEL);
 922                         return (EINVAL);
 923                 }
 924
 925                 /*
 926                  * Make sure the device hasn't shrunk.
 927                  */
 928                 if (asize < vd->vdev_asize) {
 929                         vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
 930                             VDEV_AUX_BAD_LABEL);
 931                         return (EINVAL);
 932                 }
 933
 934                 /*
 935                  * If all children are healthy and the asize has increased,
 936                  * then we've experienced dynamic LUN growth.
 937                  */
 938                 if (vd->vdev_state == VDEV_STATE_HEALTHY &&
 939                     asize > vd->vdev_asize) {
 940                         vd->vdev_asize = asize;
 941                 }
 942         }
 943
 944         /*
 945          * Ensure we can issue some IO before declaring the
 946          * vdev open for business.
 947          */
 948         error = vdev_probe(vd);
 949         if (error) {
 950                 vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
 951                     VDEV_AUX_OPEN_FAILED);
 952                 return (error);
 953         }
 954
 955         /*
 956          * If this is a top-level vdev, compute the raidz-deflation
 957          * ratio.  Note, we hard-code in 128k (1<<17) because it is the
 958          * current "typical" blocksize.  Even if SPA_MAXBLOCKSIZE
 959          * changes, this algorithm must never change, or we will
 960          * inconsistently account for existing bp's.
 961          */
 962         if (vd->vdev_top == vd) {
 963                 vd->vdev_deflate_ratio = (1<<17) /
 964                     (vdev_psize_to_asize(vd, 1<<17) >> SPA_MINBLOCKSHIFT);
 965         }
 966
 967         /*
 968          * This allows the ZFS DE to close cases appropriately.  If a device
 969          * goes away and later returns, we want to close the associated case.
 970          * But it's not enough to simply post this only when a device goes from
 971          * CANT_OPEN -> HEALTHY.  If we reboot the system and the device is
 972          * back, we also need to close the case (otherwise we will try to replay
 973          * it).  So we have to post this notifier every time.  Since this only
 974          * occurs during pool open or error recovery, this should not be an
 975          * issue.
 976          */
 977         zfs_post_ok(vd->vdev_spa, vd);
 978
 979         return (0);
 980 }
 981
 982 /*
 983  * Called once the vdevs are all opened, this routine validates the label
 984  * contents.  This needs to be done before vdev_load() so that we don't
 985  * inadvertently do repair I/Os to the wrong device.
 986  *
 987  * This function will only return failure if one of the vdevs indicates that it
 988  * has since been destroyed or exported.  This is only possible if
 989  * /etc/zfs/zpool.cache was readonly at the time.  Otherwise, the vdev state
 990  * will be updated but the function will return 0.
 991  */
 992 int
 993 vdev_validate(vdev_t *vd)
 994 {
 995         spa_t *spa = vd->vdev_spa;
 996         int c;
 997         nvlist_t *label;
 998         uint64_t guid;
 999         uint64_t state;
1000
1001         for (c = 0; c < vd->vdev_children; c++)
1002                 if (vdev_validate(vd->vdev_child[c]) != 0)
1003                         return (EBADF);
1004
1005         /*
1006          * If the device has already failed, or was marked offline, don't do
1007          * any further validation.  Otherwise, label I/O will fail and we will
1008          * overwrite the previous state.
1009          */
1010         if (vd->vdev_ops->vdev_op_leaf && !vdev_is_dead(vd)) {
1011
1012                 if ((label = vdev_label_read_config(vd)) == NULL) {
1013                         vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1014                             VDEV_AUX_BAD_LABEL);
1015                         return (0);
1016                 }
1017
1018                 if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID,
1019                     &guid) != 0 || guid != spa_guid(spa)) {
1020                         vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1021                             VDEV_AUX_CORRUPT_DATA);
1022                         nvlist_free(label);
1023                         return (0);
1024                 }
1025
1026                 if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID,
1027                     &guid) != 0 || guid != vd->vdev_guid) {
1028                         vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1029                             VDEV_AUX_CORRUPT_DATA);
1030                         nvlist_free(label);
1031                         return (0);
1032                 }
1033
1034                 if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE,
1035                     &state) != 0) {
1036                         vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1037                             VDEV_AUX_CORRUPT_DATA);
1038                         nvlist_free(label);
1039                         return (0);
1040                 }
1041
1042                 nvlist_free(label);
1043
1044                 if (spa->spa_load_state == SPA_LOAD_OPEN &&
1045                     state != POOL_STATE_ACTIVE)
1046                         return (EBADF);
1047         }
1048
1049         /*
1050          * If we were able to open and validate a vdev that was previously
1051          * marked permanently unavailable, clear that state now.
1052          */
1053         if (vd->vdev_not_present)
1054                 vd->vdev_not_present = 0;
1055
1056         return (0);
1057 }
1058
1059 /*
1060  * Close a virtual device.
1061  */
1062 void
1063 vdev_close(vdev_t *vd)
1064 {
1065         vd->vdev_ops->vdev_op_close(vd);
1066
1067         vdev_cache_purge(vd);
1068
1069         /*
1070          * We record the previous state before we close it, so  that if we are
1071          * doing a reopen(), we don't generate FMA ereports if we notice that
1072          * it's still faulted.
1073          */
1074         vd->vdev_prevstate = vd->vdev_state;
1075
1076         if (vd->vdev_offline)
1077                 vd->vdev_state = VDEV_STATE_OFFLINE;
1078         else
1079                 vd->vdev_state = VDEV_STATE_CLOSED;
1080         vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
1081 }
1082
1083 void
1084 vdev_reopen(vdev_t *vd)
1085 {
1086         spa_t *spa = vd->vdev_spa;
1087
1088         ASSERT(spa_config_held(spa, RW_WRITER));
1089
1090         vdev_close(vd);
1091         (void) vdev_open(vd);
1092
1093         /*
1094          * Call vdev_validate() here to make sure we have the same device.
1095          * Otherwise, a device with an invalid label could be successfully
1096          * opened in response to vdev_reopen().
1097          */
1098         (void) vdev_validate(vd);
1099
1100         /*
1101          * Reassess parent vdev's health.
1102          */
1103         vdev_propagate_state(vd);
1104 }
1105
1106 int
1107 vdev_create(vdev_t *vd, uint64_t txg, boolean_t isreplacing)
1108 {
1109         int error;
1110
1111         /*
1112          * Normally, partial opens (e.g. of a mirror) are allowed.
1113          * For a create, however, we want to fail the request if
1114          * there are any components we can't open.
1115          */
1116         error = vdev_open(vd);
1117
1118         if (error || vd->vdev_state != VDEV_STATE_HEALTHY) {
1119                 vdev_close(vd);
1120                 return (error ? error : ENXIO);
1121         }
1122
1123         /*
1124          * Recursively initialize all labels.
1125          */
1126         if ((error = vdev_label_init(vd, txg, isreplacing ?
1127             VDEV_LABEL_REPLACE : VDEV_LABEL_CREATE)) != 0) {
1128                 vdev_close(vd);
1129                 return (error);
1130         }
1131
1132         return (0);
1133 }
1134
1135 /*
1136  * The is the latter half of vdev_create().  It is distinct because it
1137  * involves initiating transactions in order to do metaslab creation.
1138  * For creation, we want to try to create all vdevs at once and then undo it
1139  * if anything fails; this is much harder if we have pending transactions.
1140  */
1141 void
1142 vdev_init(vdev_t *vd, uint64_t txg)
1143 {
1144         /*
1145          * Aim for roughly 200 metaslabs per vdev.
1146          */
1147         vd->vdev_ms_shift = highbit(vd->vdev_asize / 200);
1148         vd->vdev_ms_shift = MAX(vd->vdev_ms_shift, SPA_MAXBLOCKSHIFT);
1149
1150         /*
1151          * Initialize the vdev's metaslabs.  This can't fail because
1152          * there's nothing to read when creating all new metaslabs.
1153          */
1154         VERIFY(vdev_metaslab_init(vd, txg) == 0);
1155 }
1156
1157 void
1158 vdev_dirty(vdev_t *vd, int flags, void *arg, uint64_t txg)
1159 {
1160         ASSERT(vd == vd->vdev_top);
1161         ASSERT(ISP2(flags));
1162
1163         if (flags & VDD_METASLAB)
1164                 (void) txg_list_add(&vd->vdev_ms_list, arg, txg);
1165
1166         if (flags & VDD_DTL)
1167                 (void) txg_list_add(&vd->vdev_dtl_list, arg, txg);
1168
1169         (void) txg_list_add(&vd->vdev_spa->spa_vdev_txg_list, vd, txg);
1170 }
1171
1172 void
1173 vdev_dtl_dirty(space_map_t *sm, uint64_t txg, uint64_t size)
1174 {
1175         mutex_enter(sm->sm_lock);
1176         if (!space_map_contains(sm, txg, size))
1177                 space_map_add(sm, txg, size);
1178         mutex_exit(sm->sm_lock);
1179 }
1180
1181 int
1182 vdev_dtl_contains(space_map_t *sm, uint64_t txg, uint64_t size)
1183 {
1184         int dirty;
1185
1186         /*
1187          * Quick test without the lock -- covers the common case that
1188          * there are no dirty time segments.
1189          */
1190         if (sm->sm_space == 0)
1191                 return (0);
1192
1193         mutex_enter(sm->sm_lock);
1194         dirty = space_map_contains(sm, txg, size);
1195         mutex_exit(sm->sm_lock);
1196
1197         return (dirty);
1198 }
1199
1200 /*
1201  * Reassess DTLs after a config change or scrub completion.
1202  */
1203 void
1204 vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg, int scrub_done)
1205 {
1206         spa_t *spa = vd->vdev_spa;
1207         int c;
1208
1209         ASSERT(spa_config_held(spa, RW_WRITER));
1210
1211         if (vd->vdev_children == 0) {
1212                 mutex_enter(&vd->vdev_dtl_lock);
1213                 /*
1214                  * We're successfully scrubbed everything up to scrub_txg.
1215                  * Therefore, excise all old DTLs up to that point, then
1216                  * fold in the DTLs for everything we couldn't scrub.
1217                  */
1218                 if (scrub_txg != 0) {
1219                         space_map_excise(&vd->vdev_dtl_map, 0, scrub_txg);
1220                         space_map_union(&vd->vdev_dtl_map, &vd->vdev_dtl_scrub);
1221                 }
1222                 if (scrub_done)
1223                         space_map_vacate(&vd->vdev_dtl_scrub, NULL, NULL);
1224                 mutex_exit(&vd->vdev_dtl_lock);
1225                 if (txg != 0)
1226                         vdev_dirty(vd->vdev_top, VDD_DTL, vd, txg);
1227                 return;
1228         }
1229
1230         /*
1231          * Make sure the DTLs are always correct under the scrub lock.
1232          */
1233         if (vd == spa->spa_root_vdev)
1234                 mutex_enter(&spa->spa_scrub_lock);
1235
1236         mutex_enter(&vd->vdev_dtl_lock);
1237         space_map_vacate(&vd->vdev_dtl_map, NULL, NULL);
1238         space_map_vacate(&vd->vdev_dtl_scrub, NULL, NULL);
1239         mutex_exit(&vd->vdev_dtl_lock);
1240
1241         for (c = 0; c < vd->vdev_children; c++) {
1242                 vdev_t *cvd = vd->vdev_child[c];
1243                 vdev_dtl_reassess(cvd, txg, scrub_txg, scrub_done);
1244                 mutex_enter(&vd->vdev_dtl_lock);
1245                 space_map_union(&vd->vdev_dtl_map, &cvd->vdev_dtl_map);
1246                 space_map_union(&vd->vdev_dtl_scrub, &cvd->vdev_dtl_scrub);
1247                 mutex_exit(&vd->vdev_dtl_lock);
1248         }
1249
1250         if (vd == spa->spa_root_vdev)
1251                 mutex_exit(&spa->spa_scrub_lock);
1252 }
1253
1254 static int
1255 vdev_dtl_load(vdev_t *vd)
1256 {
1257         spa_t *spa = vd->vdev_spa;
1258         space_map_obj_t *smo = &vd->vdev_dtl;
1259         objset_t *mos = spa->spa_meta_objset;
1260         dmu_buf_t *db;
1261         int error;
1262
1263         ASSERT(vd->vdev_children == 0);
1264
1265         if (smo->smo_object == 0)
1266                 return (0);
1267
1268         if ((error = dmu_bonus_hold(mos, smo->smo_object, FTAG, &db)) != 0)
1269                 return (error);
1270
1271         ASSERT3U(db->db_size, >=, sizeof (*smo));
1272         bcopy(db->db_data, smo, sizeof (*smo));
1273         dmu_buf_rele(db, FTAG);
1274
1275         mutex_enter(&vd->vdev_dtl_lock);
1276         error = space_map_load(&vd->vdev_dtl_map, NULL, SM_ALLOC, smo, mos);
1277         mutex_exit(&vd->vdev_dtl_lock);
1278
1279         return (error);
1280 }
1281
1282 void
1283 vdev_dtl_sync(vdev_t *vd, uint64_t txg)
1284 {
1285         spa_t *spa = vd->vdev_spa;
1286         space_map_obj_t *smo = &vd->vdev_dtl;
1287         space_map_t *sm = &vd->vdev_dtl_map;
1288         objset_t *mos = spa->spa_meta_objset;
1289         space_map_t smsync;
1290         kmutex_t smlock;
1291         dmu_buf_t *db;
1292         dmu_tx_t *tx;
1293
1294         dprintf("%s in txg %llu pass %d\n",
1295             vdev_description(vd), (u_longlong_t)txg, spa_sync_pass(spa));
1296
1297         tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
1298
1299         if (vd->vdev_detached) {
1300                 if (smo->smo_object != 0) {
1301                         int err = dmu_object_free(mos, smo->smo_object, tx);
1302                         ASSERT3U(err, ==, 0);
1303                         smo->smo_object = 0;
1304                 }
1305                 dmu_tx_commit(tx);
1306                 dprintf("detach %s committed in txg %llu\n",
1307                     vdev_description(vd), txg);
1308                 return;
1309         }
1310
1311         if (smo->smo_object == 0) {
1312                 ASSERT(smo->smo_objsize == 0);
1313                 ASSERT(smo->smo_alloc == 0);
1314                 smo->smo_object = dmu_object_alloc(mos,
1315                     DMU_OT_SPACE_MAP, 1 << SPACE_MAP_BLOCKSHIFT,
1316                     DMU_OT_SPACE_MAP_HEADER, sizeof (*smo), tx);
1317                 ASSERT(smo->smo_object != 0);
1318                 vdev_config_dirty(vd->vdev_top);
1319         }
1320
1321         mutex_init(&smlock, NULL, MUTEX_DEFAULT, NULL);
1322
1323         space_map_create(&smsync, sm->sm_start, sm->sm_size, sm->sm_shift,
1324             &smlock);
1325
1326         mutex_enter(&smlock);
1327
1328         mutex_enter(&vd->vdev_dtl_lock);
1329         space_map_walk(sm, space_map_add, &smsync);
1330         mutex_exit(&vd->vdev_dtl_lock);
1331
1332         space_map_truncate(smo, mos, tx);
1333         space_map_sync(&smsync, SM_ALLOC, smo, mos, tx);
1334
1335         space_map_destroy(&smsync);
1336
1337         mutex_exit(&smlock);
1338         mutex_destroy(&smlock);
1339
1340         VERIFY(0 == dmu_bonus_hold(mos, smo->smo_object, FTAG, &db));
1341         dmu_buf_will_dirty(db, tx);
1342         ASSERT3U(db->db_size, >=, sizeof (*smo));
1343         bcopy(smo, db->db_data, sizeof (*smo));
1344         dmu_buf_rele(db, FTAG);
1345
1346         dmu_tx_commit(tx);
1347 }
1348
1349 void
1350 vdev_load(vdev_t *vd)
1351 {
1352         int c;
1353
1354         /*
1355          * Recursively load all children.
1356          */
1357         for (c = 0; c < vd->vdev_children; c++)
1358                 vdev_load(vd->vdev_child[c]);
1359
1360         /*
1361          * If this is a top-level vdev, initialize its metaslabs.
1362          */
1363         if (vd == vd->vdev_top &&
1364             (vd->vdev_ashift == 0 || vd->vdev_asize == 0 ||
1365             vdev_metaslab_init(vd, 0) != 0))
1366                 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1367                     VDEV_AUX_CORRUPT_DATA);
1368
1369         /*
1370          * If this is a leaf vdev, load its DTL.
1371          */
1372         if (vd->vdev_ops->vdev_op_leaf && vdev_dtl_load(vd) != 0)
1373                 vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
1374                     VDEV_AUX_CORRUPT_DATA);
1375 }
1376
1377 /*
1378  * The special vdev case is used for hot spares and l2cache devices.  Its
1379  * sole purpose it to set the vdev state for the associated vdev.  To do this,
1380  * we make sure that we can open the underlying device, then try to read the
1381  * label, and make sure that the label is sane and that it hasn't been
1382  * repurposed to another pool.
1383  */
1384 int
1385 vdev_validate_aux(vdev_t *vd)
1386 {
1387         nvlist_t *label;
1388         uint64_t guid, version;
1389         uint64_t state;
1390
1391         if ((label = vdev_label_read_config(vd)) == NULL) {
1392                 vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1393                     VDEV_AUX_CORRUPT_DATA);
1394                 return (-1);
1395         }
1396
1397         if (nvlist_lookup_uint64(label, ZPOOL_CONFIG_VERSION, &version) != 0 ||
1398             version > SPA_VERSION ||
1399             nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid) != 0 ||
1400             guid != vd->vdev_guid ||
1401             nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_STATE, &state) != 0) {
1402                 vdev_set_state(vd, B_TRUE, VDEV_STATE_CANT_OPEN,
1403                     VDEV_AUX_CORRUPT_DATA);
1404                 nvlist_free(label);
1405                 return (-1);
1406         }
1407
1408         /*
1409          * We don't actually check the pool state here.  If it's in fact in
1410          * use by another pool, we update this fact on the fly when requested.
1411          */
1412         nvlist_free(label);
1413         return (0);
1414 }
1415
1416 void
1417 vdev_sync_done(vdev_t *vd, uint64_t txg)
1418 {
1419         metaslab_t *msp;
1420
1421         dprintf("%s txg %llu\n", vdev_description(vd), txg);
1422
1423         while (msp = txg_list_remove(&vd->vdev_ms_list, TXG_CLEAN(txg)))
1424                 metaslab_sync_done(msp, txg);
1425 }
1426
1427 void
1428 vdev_sync(vdev_t *vd, uint64_t txg)
1429 {
1430         spa_t *spa = vd->vdev_spa;
1431         vdev_t *lvd;
1432         metaslab_t *msp;
1433         dmu_tx_t *tx;
1434
1435         dprintf("%s txg %llu pass %d\n",
1436             vdev_description(vd), (u_longlong_t)txg, spa_sync_pass(spa));
1437
1438         if (vd->vdev_ms_array == 0 && vd->vdev_ms_shift != 0) {
1439                 ASSERT(vd == vd->vdev_top);
1440                 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
1441                 vd->vdev_ms_array = dmu_object_alloc(spa->spa_meta_objset,
1442                     DMU_OT_OBJECT_ARRAY, 0, DMU_OT_NONE, 0, tx);
1443                 ASSERT(vd->vdev_ms_array != 0);
1444                 vdev_config_dirty(vd);
1445                 dmu_tx_commit(tx);
1446         }
1447
1448         while ((msp = txg_list_remove(&vd->vdev_ms_list, txg)) != NULL) {
1449                 metaslab_sync(msp, txg);
1450                 (void) txg_list_add(&vd->vdev_ms_list, msp, TXG_CLEAN(txg));
1451         }
1452
1453         while ((lvd = txg_list_remove(&vd->vdev_dtl_list, txg)) != NULL)
1454                 vdev_dtl_sync(lvd, txg);
1455
1456         (void) txg_list_add(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg));
1457 }
1458
1459 uint64_t
1460 vdev_psize_to_asize(vdev_t *vd, uint64_t psize)
1461 {
1462         return (vd->vdev_ops->vdev_op_asize(vd, psize));
1463 }
1464
1465 const char *
1466 vdev_description(vdev_t *vd)
1467 {
1468         if (vd == NULL || vd->vdev_ops == NULL)
1469                 return ("<unknown>");
1470
1471         if (vd->vdev_path != NULL)
1472                 return (vd->vdev_path);
1473
1474         if (vd->vdev_parent == NULL)
1475                 return (spa_name(vd->vdev_spa));
1476
1477         return (vd->vdev_ops->vdev_op_type);
1478 }
1479
1480 /*
1481  * Mark the given vdev faulted.  A faulted vdev behaves as if the device could
1482  * not be opened, and no I/O is attempted.
1483  */
1484 int
1485 vdev_fault(spa_t *spa, uint64_t guid)
1486 {
1487         vdev_t *rvd, *vd;
1488         uint64_t txg;
1489
1490         /*
1491          * Disregard a vdev fault request if the pool has
1492          * experienced a complete failure.
1493          *
1494          * XXX - We do this here so that we don't hold the
1495          * spa_namespace_lock in the event that we can't get
1496          * the RW_WRITER spa_config_lock.
1497          */
1498         if (spa_state(spa) == POOL_STATE_IO_FAILURE)
1499                 return (EIO);
1500
1501         txg = spa_vdev_enter(spa);
1502
1503         rvd = spa->spa_root_vdev;
1504
1505         if ((vd = vdev_lookup_by_guid(rvd, guid)) == NULL)
1506                 return (spa_vdev_exit(spa, NULL, txg, ENODEV));
1507         if (!vd->vdev_ops->vdev_op_leaf)
1508                 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1509
1510         /*
1511          * Faulted state takes precedence over degraded.
1512          */
1513         vd->vdev_faulted = 1ULL;
1514         vd->vdev_degraded = 0ULL;
1515         vdev_set_state(vd, B_FALSE, VDEV_STATE_FAULTED,
1516             VDEV_AUX_ERR_EXCEEDED);
1517
1518         /*
1519          * If marking the vdev as faulted cause the toplevel vdev to become
1520          * unavailable, then back off and simply mark the vdev as degraded
1521          * instead.
1522          */
1523         if (vdev_is_dead(vd->vdev_top)) {
1524                 vd->vdev_degraded = 1ULL;
1525                 vd->vdev_faulted = 0ULL;
1526
1527                 /*
1528                  * If we reopen the device and it's not dead, only then do we
1529                  * mark it degraded.
1530                  */
1531                 vdev_reopen(vd);
1532
1533                 if (vdev_readable(vd)) {
1534                         vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED,
1535                             VDEV_AUX_ERR_EXCEEDED);
1536                 }
1537         }
1538
1539         vdev_config_dirty(vd->vdev_top);
1540
1541         (void) spa_vdev_exit(spa, NULL, txg, 0);
1542
1543         return (0);
1544 }
1545
1546 /*
1547  * Mark the given vdev degraded.  A degraded vdev is purely an indication to the
1548  * user that something is wrong.  The vdev continues to operate as normal as far
1549  * as I/O is concerned.
1550  */
1551 int
1552 vdev_degrade(spa_t *spa, uint64_t guid)
1553 {
1554         vdev_t *rvd, *vd;
1555         uint64_t txg;
1556
1557         /*
1558          * Disregard a vdev fault request if the pool has
1559          * experienced a complete failure.
1560          *
1561          * XXX - We do this here so that we don't hold the
1562          * spa_namespace_lock in the event that we can't get
1563          * the RW_WRITER spa_config_lock.
1564          */
1565         if (spa_state(spa) == POOL_STATE_IO_FAILURE)
1566                 return (EIO);
1567
1568         txg = spa_vdev_enter(spa);
1569
1570         rvd = spa->spa_root_vdev;
1571
1572         if ((vd = vdev_lookup_by_guid(rvd, guid)) == NULL)
1573                 return (spa_vdev_exit(spa, NULL, txg, ENODEV));
1574         if (!vd->vdev_ops->vdev_op_leaf)
1575                 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1576
1577         /*
1578          * If the vdev is already faulted, then don't do anything.
1579          */
1580         if (vd->vdev_faulted || vd->vdev_degraded) {
1581                 (void) spa_vdev_exit(spa, NULL, txg, 0);
1582                 return (0);
1583         }
1584
1585         vd->vdev_degraded = 1ULL;
1586         if (!vdev_is_dead(vd))
1587                 vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED,
1588                     VDEV_AUX_ERR_EXCEEDED);
1589         vdev_config_dirty(vd->vdev_top);
1590
1591         (void) spa_vdev_exit(spa, NULL, txg, 0);
1592
1593         return (0);
1594 }
1595
1596 /*
1597  * Online the given vdev.  If 'unspare' is set, it implies two things.  First,
1598  * any attached spare device should be detached when the device finishes
1599  * resilvering.  Second, the online should be treated like a 'test' online case,
1600  * so no FMA events are generated if the device fails to open.
1601  */
1602 int
1603 vdev_online(spa_t *spa, uint64_t guid, uint64_t flags,
1604     vdev_state_t *newstate)
1605 {
1606         vdev_t *rvd, *vd;
1607         uint64_t txg;
1608
1609         /*
1610          * Disregard a vdev fault request if the pool has
1611          * experienced a complete failure.
1612          *
1613          * XXX - We do this here so that we don't hold the
1614          * spa_namespace_lock in the event that we can't get
1615          * the RW_WRITER spa_config_lock.
1616          */
1617         if (spa_state(spa) == POOL_STATE_IO_FAILURE)
1618                 return (EIO);
1619
1620         txg = spa_vdev_enter(spa);
1621
1622         rvd = spa->spa_root_vdev;
1623
1624         if ((vd = vdev_lookup_by_guid(rvd, guid)) == NULL)
1625                 return (spa_vdev_exit(spa, NULL, txg, ENODEV));
1626
1627         if (!vd->vdev_ops->vdev_op_leaf)
1628                 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1629
1630         vd->vdev_offline = B_FALSE;
1631         vd->vdev_tmpoffline = B_FALSE;
1632         vd->vdev_checkremove = (flags & ZFS_ONLINE_CHECKREMOVE) ?
1633             B_TRUE : B_FALSE;
1634         vd->vdev_forcefault = (flags & ZFS_ONLINE_FORCEFAULT) ?
1635             B_TRUE : B_FALSE;
1636         vdev_reopen(vd->vdev_top);
1637         vd->vdev_checkremove = vd->vdev_forcefault = B_FALSE;
1638
1639         if (newstate)
1640                 *newstate = vd->vdev_state;
1641         if ((flags & ZFS_ONLINE_UNSPARE) &&
1642             !vdev_is_dead(vd) && vd->vdev_parent &&
1643             vd->vdev_parent->vdev_ops == &vdev_spare_ops &&
1644             vd->vdev_parent->vdev_child[0] == vd)
1645                 vd->vdev_unspare = B_TRUE;
1646
1647         vdev_config_dirty(vd->vdev_top);
1648
1649         (void) spa_vdev_exit(spa, NULL, txg, 0);
1650
1651         /*
1652          * Must hold spa_namespace_lock in order to post resilver sysevent
1653          * w/pool name.
1654          */
1655         mutex_enter(&spa_namespace_lock);
1656         VERIFY(spa_scrub(spa, POOL_SCRUB_RESILVER, B_TRUE) == 0);
1657         mutex_exit(&spa_namespace_lock);
1658
1659         return (0);
1660 }
1661
1662 int
1663 vdev_offline(spa_t *spa, uint64_t guid, uint64_t flags)
1664 {
1665         vdev_t *rvd, *vd;
1666         uint64_t txg;
1667
1668         /*
1669          * Disregard a vdev fault request if the pool has
1670          * experienced a complete failure.
1671          *
1672          * XXX - We do this here so that we don't hold the
1673          * spa_namespace_lock in the event that we can't get
1674          * the RW_WRITER spa_config_lock.
1675          */
1676         if (spa_state(spa) == POOL_STATE_IO_FAILURE)
1677                 return (EIO);
1678
1679         txg = spa_vdev_enter(spa);
1680
1681         rvd = spa->spa_root_vdev;
1682
1683         if ((vd = vdev_lookup_by_guid(rvd, guid)) == NULL)
1684                 return (spa_vdev_exit(spa, NULL, txg, ENODEV));
1685
1686         if (!vd->vdev_ops->vdev_op_leaf)
1687                 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP));
1688
1689         /*
1690          * If the device isn't already offline, try to offline it.
1691          */
1692         if (!vd->vdev_offline) {
1693                 /*
1694                  * If this device's top-level vdev has a non-empty DTL,
1695                  * don't allow the device to be offlined.
1696                  *
1697                  * XXX -- make this more precise by allowing the offline
1698                  * as long as the remaining devices don't have any DTL holes.
1699                  */
1700                 if (vd->vdev_top->vdev_dtl_map.sm_space != 0)
1701                         return (spa_vdev_exit(spa, NULL, txg, EBUSY));
1702
1703                 /*
1704                  * Offline this device and reopen its top-level vdev.
1705                  * If this action results in the top-level vdev becoming
1706                  * unusable, undo it and fail the request.
1707                  */
1708                 vd->vdev_offline = B_TRUE;
1709                 vdev_reopen(vd->vdev_top);
1710                 if (vdev_is_dead(vd->vdev_top)) {
1711                         vd->vdev_offline = B_FALSE;
1712                         vdev_reopen(vd->vdev_top);
1713                         return (spa_vdev_exit(spa, NULL, txg, EBUSY));
1714                 }
1715         }
1716
1717         vd->vdev_tmpoffline = (flags & ZFS_OFFLINE_TEMPORARY) ?
1718             B_TRUE : B_FALSE;
1719
1720         vdev_config_dirty(vd->vdev_top);
1721
1722         return (spa_vdev_exit(spa, NULL, txg, 0));
1723 }
1724
1725 /*
1726  * Clear the error counts associated with this vdev.  Unlike vdev_online() and
1727  * vdev_offline(), we assume the spa config is locked.  We also clear all
1728  * children.  If 'vd' is NULL, then the user wants to clear all vdevs.
1729  * If reopen is specified then attempt to reopen the vdev if the vdev is
1730  * faulted or degraded.
1731  */
1732 void
1733 vdev_clear(spa_t *spa, vdev_t *vd, boolean_t reopen_wanted)
1734 {
1735         int c;
1736
1737         if (vd == NULL)
1738                 vd = spa->spa_root_vdev;
1739
1740         vd->vdev_stat.vs_read_errors = 0;
1741         vd->vdev_stat.vs_write_errors = 0;
1742         vd->vdev_stat.vs_checksum_errors = 0;
1743         vd->vdev_is_failing = B_FALSE;
1744
1745         for (c = 0; c < vd->vdev_children; c++)
1746                 vdev_clear(spa, vd->vdev_child[c], reopen_wanted);
1747
1748         /*
1749          * If we're in the FAULTED state, then clear the persistent state and
1750          * attempt to reopen the device.  We also mark the vdev config dirty, so
1751          * that the new faulted state is written out to disk.
1752          */
1753         if (reopen_wanted && (vd->vdev_faulted || vd->vdev_degraded)) {
1754                 vd->vdev_faulted = vd->vdev_degraded = 0;
1755                 vdev_reopen(vd);
1756                 vdev_config_dirty(vd->vdev_top);
1757
1758                 if (vd->vdev_faulted)
1759                         spa_async_request(spa, SPA_ASYNC_RESILVER);
1760
1761                 spa_event_notify(spa, vd, ESC_ZFS_VDEV_CLEAR);
1762         }
1763 }
1764
1765 int
1766 vdev_readable(vdev_t *vd)
1767 {
1768         /* XXPOLICY */
1769         return (!vdev_is_dead(vd));
1770 }
1771
1772 int
1773 vdev_writeable(vdev_t *vd)
1774 {
1775         return (!vdev_is_dead(vd) && !vd->vdev_is_failing);
1776 }
1777
1778 int
1779 vdev_is_dead(vdev_t *vd)
1780 {
1781         return (vd->vdev_state < VDEV_STATE_DEGRADED);
1782 }
1783
1784 int
1785 vdev_error_inject(vdev_t *vd, zio_t *zio)
1786 {
1787         int error = 0;
1788
1789         if (vd->vdev_fault_mode == VDEV_FAULT_NONE)
1790                 return (0);
1791
1792         if (((1ULL << zio->io_type) & vd->vdev_fault_mask) == 0)
1793                 return (0);
1794
1795         switch (vd->vdev_fault_mode) {
1796         case VDEV_FAULT_RANDOM:
1797                 if (spa_get_random(vd->vdev_fault_arg) == 0)
1798                         error = EIO;
1799                 break;
1800
1801         case VDEV_FAULT_COUNT:
1802                 if ((int64_t)--vd->vdev_fault_arg <= 0)
1803                         vd->vdev_fault_mode = VDEV_FAULT_NONE;
1804                 error = EIO;
1805                 break;
1806         }
1807
1808         return (error);
1809 }
1810
1811 /*
1812  * Get statistics for the given vdev.
1813  */
1814 void
1815 vdev_get_stats(vdev_t *vd, vdev_stat_t *vs)
1816 {
1817         vdev_t *rvd = vd->vdev_spa->spa_root_vdev;
1818         int c, t;
1819
1820         mutex_enter(&vd->vdev_stat_lock);
1821         bcopy(&vd->vdev_stat, vs, sizeof (*vs));
1822         vs->vs_timestamp = gethrtime() - vs->vs_timestamp;
1823         vs->vs_state = vd->vdev_state;
1824         vs->vs_rsize = vdev_get_rsize(vd);
1825         mutex_exit(&vd->vdev_stat_lock);
1826
1827         /*
1828          * If we're getting stats on the root vdev, aggregate the I/O counts
1829          * over all top-level vdevs (i.e. the direct children of the root).
1830          */
1831         if (vd == rvd) {
1832                 for (c = 0; c < rvd->vdev_children; c++) {
1833                         vdev_t *cvd = rvd->vdev_child[c];
1834                         vdev_stat_t *cvs = &cvd->vdev_stat;
1835
1836                         mutex_enter(&vd->vdev_stat_lock);
1837                         for (t = 0; t < ZIO_TYPES; t++) {
1838                                 vs->vs_ops[t] += cvs->vs_ops[t];
1839                                 vs->vs_bytes[t] += cvs->vs_bytes[t];
1840                         }
1841                         vs->vs_read_errors += cvs->vs_read_errors;
1842                         vs->vs_write_errors += cvs->vs_write_errors;
1843                         vs->vs_checksum_errors += cvs->vs_checksum_errors;
1844                         vs->vs_scrub_examined += cvs->vs_scrub_examined;
1845                         vs->vs_scrub_errors += cvs->vs_scrub_errors;
1846                         mutex_exit(&vd->vdev_stat_lock);
1847                 }
1848         }
1849 }
1850
1851 void
1852 vdev_clear_stats(vdev_t *vd)
1853 {
1854         mutex_enter(&vd->vdev_stat_lock);
1855         vd->vdev_stat.vs_space = 0;
1856         vd->vdev_stat.vs_dspace = 0;
1857         vd->vdev_stat.vs_alloc = 0;
1858         mutex_exit(&vd->vdev_stat_lock);
1859 }
1860
1861 void
1862 vdev_stat_update(zio_t *zio)
1863 {
1864         vdev_t *vd = zio->io_vd;
1865         vdev_t *pvd;
1866         uint64_t txg = zio->io_txg;
1867         vdev_stat_t *vs = &vd->vdev_stat;
1868         zio_type_t type = zio->io_type;
1869         int flags = zio->io_flags;
1870
1871         if (zio->io_error == 0) {
1872                 if (!(flags & ZIO_FLAG_IO_BYPASS)) {
1873                         mutex_enter(&vd->vdev_stat_lock);
1874                         vs->vs_ops[type]++;
1875                         vs->vs_bytes[type] += zio->io_size;
1876                         mutex_exit(&vd->vdev_stat_lock);
1877                 }
1878                 if ((flags & ZIO_FLAG_IO_REPAIR) &&
1879                     zio->io_delegate_list == NULL) {
1880                         mutex_enter(&vd->vdev_stat_lock);
1881                         if (flags & ZIO_FLAG_SCRUB_THREAD)
1882                                 vs->vs_scrub_repaired += zio->io_size;
1883                         else
1884                                 vs->vs_self_healed += zio->io_size;
1885                         mutex_exit(&vd->vdev_stat_lock);
1886                 }
1887                 return;
1888         }
1889
1890         if (flags & ZIO_FLAG_SPECULATIVE)
1891                 return;
1892
1893         if (vdev_readable(vd)) {
1894                 mutex_enter(&vd->vdev_stat_lock);
1895                 if (type == ZIO_TYPE_READ) {
1896                         if (zio->io_error == ECKSUM)
1897                                 vs->vs_checksum_errors++;
1898                         else
1899                                 vs->vs_read_errors++;
1900                 }
1901                 if (type == ZIO_TYPE_WRITE)
1902                         vs->vs_write_errors++;
1903                 mutex_exit(&vd->vdev_stat_lock);
1904         }
1905
1906         if (type == ZIO_TYPE_WRITE) {
1907                 if (txg == 0 || vd->vdev_children != 0)
1908                         return;
1909                 if (flags & ZIO_FLAG_SCRUB_THREAD) {
1910                         ASSERT(flags & ZIO_FLAG_IO_REPAIR);
1911                         for (pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)
1912                                 vdev_dtl_dirty(&pvd->vdev_dtl_scrub, txg, 1);
1913                 }
1914                 if (!(flags & ZIO_FLAG_IO_REPAIR)) {
1915                         if (vdev_dtl_contains(&vd->vdev_dtl_map, txg, 1))
1916                                 return;
1917                         vdev_dirty(vd->vdev_top, VDD_DTL, vd, txg);
1918                         for (pvd = vd; pvd != NULL; pvd = pvd->vdev_parent)
1919                                 vdev_dtl_dirty(&pvd->vdev_dtl_map, txg, 1);
1920                 }
1921         }
1922 }
1923
1924 void
1925 vdev_scrub_stat_update(vdev_t *vd, pool_scrub_type_t type, boolean_t complete)
1926 {
1927         int c;
1928         vdev_stat_t *vs = &vd->vdev_stat;
1929
1930         for (c = 0; c < vd->vdev_children; c++)
1931                 vdev_scrub_stat_update(vd->vdev_child[c], type, complete);
1932
1933         mutex_enter(&vd->vdev_stat_lock);
1934
1935         if (type == POOL_SCRUB_NONE) {
1936                 /*
1937                  * Update completion and end time.  Leave everything else alone
1938                  * so we can report what happened during the previous scrub.
1939                  */
1940                 vs->vs_scrub_complete = complete;
1941                 vs->vs_scrub_end = gethrestime_sec();
1942         } else {
1943                 vs->vs_scrub_type = type;
1944                 vs->vs_scrub_complete = 0;
1945                 vs->vs_scrub_examined = 0;
1946                 vs->vs_scrub_repaired = 0;
1947                 vs->vs_scrub_errors = 0;
1948                 vs->vs_scrub_start = gethrestime_sec();
1949                 vs->vs_scrub_end = 0;
1950         }
1951
1952         mutex_exit(&vd->vdev_stat_lock);
1953 }
1954
1955 /*
1956  * Update the in-core space usage stats for this vdev and the root vdev.
1957  */
1958 void
1959 vdev_space_update(vdev_t *vd, int64_t space_delta, int64_t alloc_delta,
1960     boolean_t update_root)
1961 {
1962         int64_t dspace_delta = space_delta;
1963         spa_t *spa = vd->vdev_spa;
1964         vdev_t *rvd = spa->spa_root_vdev;
1965
1966         ASSERT(vd == vd->vdev_top);
1967
1968         /*
1969          * Apply the inverse of the psize-to-asize (ie. RAID-Z) space-expansion
1970          * factor.  We must calculate this here and not at the root vdev
1971          * because the root vdev's psize-to-asize is simply the max of its
1972          * childrens', thus not accurate enough for us.
1973          */
1974         ASSERT((dspace_delta & (SPA_MINBLOCKSIZE-1)) == 0);
1975         dspace_delta = (dspace_delta >> SPA_MINBLOCKSHIFT) *
1976             vd->vdev_deflate_ratio;
1977
1978         mutex_enter(&vd->vdev_stat_lock);
1979         vd->vdev_stat.vs_space += space_delta;
1980         vd->vdev_stat.vs_alloc += alloc_delta;
1981         vd->vdev_stat.vs_dspace += dspace_delta;
1982         mutex_exit(&vd->vdev_stat_lock);
1983
1984         if (update_root) {
1985                 ASSERT(rvd == vd->vdev_parent);
1986                 ASSERT(vd->vdev_ms_count != 0);
1987
1988                 /*
1989                  * Don't count non-normal (e.g. intent log) space as part of
1990                  * the pool's capacity.
1991                  */
1992                 if (vd->vdev_mg->mg_class != spa->spa_normal_class)
1993                         return;
1994
1995                 mutex_enter(&rvd->vdev_stat_lock);
1996                 rvd->vdev_stat.vs_space += space_delta;
1997                 rvd->vdev_stat.vs_alloc += alloc_delta;
1998                 rvd->vdev_stat.vs_dspace += dspace_delta;
1999                 mutex_exit(&rvd->vdev_stat_lock);
2000         }
2001 }
2002
2003 /*
2004  * Mark a top-level vdev's config as dirty, placing it on the dirty list
2005  * so that it will be written out next time the vdev configuration is synced.
2006  * If the root vdev is specified (vdev_top == NULL), dirty all top-level vdevs.
2007  */
2008 void
2009 vdev_config_dirty(vdev_t *vd)
2010 {
2011         spa_t *spa = vd->vdev_spa;
2012         vdev_t *rvd = spa->spa_root_vdev;
2013         int c;
2014
2015         /*
2016          * The dirty list is protected by the config lock.  The caller must
2017          * either hold the config lock as writer, or must be the sync thread
2018          * (which holds the lock as reader).  There's only one sync thread,
2019          * so this is sufficient to ensure mutual exclusion.
2020          */
2021         ASSERT(spa_config_held(spa, RW_WRITER) ||
2022             dsl_pool_sync_context(spa_get_dsl(spa)));
2023
2024         if (vd == rvd) {
2025                 for (c = 0; c < rvd->vdev_children; c++)
2026                         vdev_config_dirty(rvd->vdev_child[c]);
2027         } else {
2028                 ASSERT(vd == vd->vdev_top);
2029
2030                 if (!list_link_active(&vd->vdev_dirty_node))
2031                         list_insert_head(&spa->spa_dirty_list, vd);
2032         }
2033 }
2034
2035 void
2036 vdev_config_clean(vdev_t *vd)
2037 {
2038         spa_t *spa = vd->vdev_spa;
2039
2040         ASSERT(spa_config_held(spa, RW_WRITER) ||
2041             dsl_pool_sync_context(spa_get_dsl(spa)));
2042
2043         ASSERT(list_link_active(&vd->vdev_dirty_node));
2044         list_remove(&spa->spa_dirty_list, vd);
2045 }
2046
2047 void
2048 vdev_propagate_state(vdev_t *vd)
2049 {
2050         vdev_t *rvd = vd->vdev_spa->spa_root_vdev;
2051         int degraded = 0, faulted = 0;
2052         int corrupted = 0;
2053         int c;
2054         vdev_t *child;
2055
2056         if (vd->vdev_children > 0) {
2057                 for (c = 0; c < vd->vdev_children; c++) {
2058                         child = vd->vdev_child[c];
2059                         if (vdev_is_dead(child) && !vdev_readable(child))
2060                                 faulted++;
2061                         else if (child->vdev_state <= VDEV_STATE_DEGRADED)
2062                                 degraded++;
2063
2064                         if (child->vdev_stat.vs_aux == VDEV_AUX_CORRUPT_DATA)
2065                                 corrupted++;
2066                 }
2067
2068                 vd->vdev_ops->vdev_op_state_change(vd, faulted, degraded);
2069
2070                 /*
2071                  * Root special: if there is a toplevel vdev that cannot be
2072                  * opened due to corrupted metadata, then propagate the root
2073                  * vdev's aux state as 'corrupt' rather than 'insufficient
2074                  * replicas'.
2075                  */
2076                 if (corrupted && vd == rvd &&
2077                     rvd->vdev_state == VDEV_STATE_CANT_OPEN)
2078                         vdev_set_state(rvd, B_FALSE, VDEV_STATE_CANT_OPEN,
2079                             VDEV_AUX_CORRUPT_DATA);
2080         }
2081
2082         if (vd->vdev_parent && !vd->vdev_islog)
2083                 vdev_propagate_state(vd->vdev_parent);
2084 }
2085
2086 /*
2087  * Set a vdev's state.  If this is during an open, we don't update the parent
2088  * state, because we're in the process of opening children depth-first.
2089  * Otherwise, we propagate the change to the parent.
2090  *
2091  * If this routine places a device in a faulted state, an appropriate ereport is
2092  * generated.
2093  */
2094 void
2095 vdev_set_state(vdev_t *vd, boolean_t isopen, vdev_state_t state, vdev_aux_t aux)
2096 {
2097         uint64_t save_state;
2098
2099         if (state == vd->vdev_state) {
2100                 vd->vdev_stat.vs_aux = aux;
2101                 return;
2102         }
2103
2104         save_state = vd->vdev_state;
2105
2106         vd->vdev_state = state;
2107         vd->vdev_stat.vs_aux = aux;
2108
2109         /*
2110          * If we are setting the vdev state to anything but an open state, then
2111          * always close the underlying device.  Otherwise, we keep accessible
2112          * but invalid devices open forever.  We don't call vdev_close() itself,
2113          * because that implies some extra checks (offline, etc) that we don't
2114          * want here.  This is limited to leaf devices, because otherwise
2115          * closing the device will affect other children.
2116          */
2117         if (!vdev_readable(vd) && vd->vdev_ops->vdev_op_leaf)
2118                 vd->vdev_ops->vdev_op_close(vd);
2119
2120         if (vd->vdev_removed &&
2121             state == VDEV_STATE_CANT_OPEN &&
2122             (aux == VDEV_AUX_OPEN_FAILED || vd->vdev_checkremove)) {
2123                 /*
2124                  * If the previous state is set to VDEV_STATE_REMOVED, then this
2125                  * device was previously marked removed and someone attempted to
2126                  * reopen it.  If this failed due to a nonexistent device, then
2127                  * keep the device in the REMOVED state.  We also let this be if
2128                  * it is one of our special test online cases, which is only
2129                  * attempting to online the device and shouldn't generate an FMA
2130                  * fault.
2131                  */
2132                 vd->vdev_state = VDEV_STATE_REMOVED;
2133                 vd->vdev_stat.vs_aux = VDEV_AUX_NONE;
2134         } else if (state == VDEV_STATE_REMOVED) {
2135                 /*
2136                  * Indicate to the ZFS DE that this device has been removed, and
2137                  * any recent errors should be ignored.
2138                  */
2139                 zfs_post_remove(vd->vdev_spa, vd);
2140                 vd->vdev_removed = B_TRUE;
2141         } else if (state == VDEV_STATE_CANT_OPEN) {
2142                 /*
2143                  * If we fail to open a vdev during an import, we mark it as
2144                  * "not available", which signifies that it was never there to
2145                  * begin with.  Failure to open such a device is not considered
2146                  * an error.
2147                  */
2148                 if (vd->vdev_spa->spa_load_state == SPA_LOAD_IMPORT &&
2149                     vd->vdev_ops->vdev_op_leaf)
2150                         vd->vdev_not_present = 1;
2151
2152                 /*
2153                  * Post the appropriate ereport.  If the 'prevstate' field is
2154                  * set to something other than VDEV_STATE_UNKNOWN, it indicates
2155                  * that this is part of a vdev_reopen().  In this case, we don't
2156                  * want to post the ereport if the device was already in the
2157                  * CANT_OPEN state beforehand.
2158                  *
2159                  * If the 'checkremove' flag is set, then this is an attempt to
2160                  * online the device in response to an insertion event.  If we
2161                  * hit this case, then we have detected an insertion event for a
2162                  * faulted or offline device that wasn't in the removed state.
2163                  * In this scenario, we don't post an ereport because we are
2164                  * about to replace the device, or attempt an online with
2165                  * vdev_forcefault, which will generate the fault for us.
2166                  */
2167                 if ((vd->vdev_prevstate != state || vd->vdev_forcefault) &&
2168                     !vd->vdev_not_present && !vd->vdev_checkremove &&
2169                     vd != vd->vdev_spa->spa_root_vdev) {
2170                         const char *class;
2171
2172                         switch (aux) {
2173                         case VDEV_AUX_OPEN_FAILED:
2174                                 class = FM_EREPORT_ZFS_DEVICE_OPEN_FAILED;
2175                                 break;
2176                         case VDEV_AUX_CORRUPT_DATA:
2177                                 class = FM_EREPORT_ZFS_DEVICE_CORRUPT_DATA;
2178                                 break;
2179                         case VDEV_AUX_NO_REPLICAS:
2180                                 class = FM_EREPORT_ZFS_DEVICE_NO_REPLICAS;
2181                                 break;
2182                         case VDEV_AUX_BAD_GUID_SUM:
2183                                 class = FM_EREPORT_ZFS_DEVICE_BAD_GUID_SUM;
2184                                 break;
2185                         case VDEV_AUX_TOO_SMALL:
2186                                 class = FM_EREPORT_ZFS_DEVICE_TOO_SMALL;
2187                                 break;
2188                         case VDEV_AUX_BAD_LABEL:
2189                                 class = FM_EREPORT_ZFS_DEVICE_BAD_LABEL;
2190                                 break;
2191                         default:
2192                                 class = FM_EREPORT_ZFS_DEVICE_UNKNOWN;
2193                         }
2194
2195                         zfs_ereport_post(class, vd->vdev_spa,
2196                             vd, NULL, save_state, 0);
2197                 }
2198
2199                 /* Erase any notion of persistent removed state */
2200                 vd->vdev_removed = B_FALSE;
2201         } else {
2202                 vd->vdev_removed = B_FALSE;
2203         }
2204
2205         if (!isopen)
2206                 vdev_propagate_state(vd);
2207 }