Explicitly flush output at end of each zevent
[zfs.git] / cmd / zpool / zpool_main.c
index cb0535a..19f1f28 100644 (file)
@@ -54,6 +54,7 @@
 
 #include "zpool_util.h"
 #include "zfs_comutil.h"
+#include "zfeature_common.h"
 
 #include "statcommon.h"
 
@@ -208,7 +209,7 @@ get_usage(zpool_help_t idx) {
        case HELP_CLEAR:
                return (gettext("\tclear [-nF] <pool> [device]\n"));
        case HELP_CREATE:
-               return (gettext("\tcreate [-fn] [-o property=value] ... \n"
+               return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
                    "\t    [-O file-system-property=value] ... \n"
                    "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
        case HELP_DESTROY:
@@ -245,7 +246,7 @@ get_usage(zpool_help_t idx) {
        case HELP_REMOVE:
                return (gettext("\tremove <pool> <device> ...\n"));
        case HELP_REOPEN:
-               return (""); /* Undocumented command */
+               return (gettext("\treopen <pool>\n"));
        case HELP_SCRUB:
                return (gettext("\tscrub [-s] <pool> ...\n"));
        case HELP_STATUS:
@@ -341,6 +342,12 @@ usage(boolean_t requested)
                /* Iterate over all properties */
                (void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
                    ZFS_TYPE_POOL);
+
+               (void) fprintf(fp, "\t%-15s   ", "feature@...");
+               (void) fprintf(fp, "YES   disabled | enabled | active\n");
+
+               (void) fprintf(fp, gettext("\nThe feature@ properties must be "
+                   "appended with a feature name.\nSee zpool-features(5).\n"));
        }
 
        /*
@@ -384,6 +391,18 @@ print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
        }
 }
 
+static boolean_t
+prop_list_contains_feature(nvlist_t *proplist)
+{
+       nvpair_t *nvp;
+       for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
+           nvp = nvlist_next_nvpair(proplist, nvp)) {
+               if (zpool_prop_feature(nvpair_name(nvp)))
+                       return (B_TRUE);
+       }
+       return (B_FALSE);
+}
+
 /*
  * Add a property pair (name, string-value) into a property nvlist.
  */
@@ -407,12 +426,34 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
        proplist = *props;
 
        if (poolprop) {
-               if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL) {
+               const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
+
+               if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL &&
+                   !zpool_prop_feature(propname)) {
                        (void) fprintf(stderr, gettext("property '%s' is "
                            "not a valid pool property\n"), propname);
                        return (2);
                }
-               normnm = zpool_prop_to_name(prop);
+
+               /*
+                * feature@ properties and version should not be specified
+                * at the same time.
+                */
+               if ((prop == ZPROP_INVAL && zpool_prop_feature(propname) &&
+                   nvlist_exists(proplist, vname)) ||
+                   (prop == ZPOOL_PROP_VERSION &&
+                   prop_list_contains_feature(proplist))) {
+                       (void) fprintf(stderr, gettext("'feature@' and "
+                           "'version' properties cannot be specified "
+                           "together\n"));
+                       return (2);
+               }
+
+
+               if (zpool_prop_feature(propname))
+                       normnm = propname;
+               else
+                       normnm = zpool_prop_to_name(prop);
        } else {
                if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
                        normnm = zfs_prop_to_name(fprop);
@@ -601,7 +642,7 @@ zpool_do_remove(int argc, char **argv)
 }
 
 /*
- * zpool create [-fn] [-o property=value] ...
+ * zpool create [-fnd] [-o property=value] ...
  *             [-O file-system-property=value] ...
  *             [-R root] [-m mountpoint] <pool> <dev> ...
  *
@@ -610,8 +651,10 @@ zpool_do_remove(int argc, char **argv)
  *             were to be created.
  *      -R     Create a pool under an alternate root
  *      -m     Set default mountpoint for the root dataset.  By default it's
- *             '/<pool>'
+ *             '/<pool>'
  *     -o      Set property=value.
+ *     -d      Don't automatically enable all supported pool features
+ *             (individual features can be enabled with -o).
  *     -O      Set fsproperty=value in the pool's root file system
  *
  * Creates the named pool according to the given vdev specification.  The
@@ -624,6 +667,7 @@ zpool_do_create(int argc, char **argv)
 {
        boolean_t force = B_FALSE;
        boolean_t dryrun = B_FALSE;
+       boolean_t enable_all_pool_feat = B_TRUE;
        int c;
        nvlist_t *nvroot = NULL;
        char *poolname;
@@ -635,7 +679,7 @@ zpool_do_create(int argc, char **argv)
        char *propval;
 
        /* check options */
-       while ((c = getopt(argc, argv, ":fnR:m:o:O:")) != -1) {
+       while ((c = getopt(argc, argv, ":fndR:m:o:O:")) != -1) {
                switch (c) {
                case 'f':
                        force = B_TRUE;
@@ -643,6 +687,9 @@ zpool_do_create(int argc, char **argv)
                case 'n':
                        dryrun = B_TRUE;
                        break;
+               case 'd':
+                       enable_all_pool_feat = B_FALSE;
+                       break;
                case 'R':
                        altroot = optarg;
                        if (add_prop_list(zpool_prop_to_name(
@@ -670,6 +717,21 @@ zpool_do_create(int argc, char **argv)
 
                        if (add_prop_list(optarg, propval, &props, B_TRUE))
                                goto errout;
+
+                       /*
+                        * If the user is creating a pool that doesn't support
+                        * feature flags, don't enable any features.
+                        */
+                       if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
+                               char *end;
+                               u_longlong_t ver;
+
+                               ver = strtoull(propval, &end, 10);
+                               if (*end == '\0' &&
+                                   ver < SPA_VERSION_FEATURES) {
+                                       enable_all_pool_feat = B_FALSE;
+                               }
+                       }
                        break;
                case 'O':
                        if ((propval = strchr(optarg, '=')) == NULL) {
@@ -735,7 +797,6 @@ zpool_do_create(int argc, char **argv)
                goto errout;
        }
 
-
        if (altroot != NULL && altroot[0] != '/') {
                (void) fprintf(stderr, gettext("invalid alternate root '%s': "
                    "must be an absolute path\n"), altroot);
@@ -817,6 +878,27 @@ zpool_do_create(int argc, char **argv)
                /*
                 * Hand off to libzfs.
                 */
+               if (enable_all_pool_feat) {
+                       int i;
+                       for (i = 0; i < SPA_FEATURES; i++) {
+                               char propname[MAXPATHLEN];
+                               zfeature_info_t *feat = &spa_feature_table[i];
+
+                               (void) snprintf(propname, sizeof (propname),
+                                   "feature@%s", feat->fi_uname);
+
+                               /*
+                                * Skip feature if user specified it manually
+                                * on the command line.
+                                */
+                               if (nvlist_exists(props, propname))
+                                       continue;
+
+                               if (add_prop_list(propname, ZFS_FEATURE_ENABLED,
+                                   &props, B_TRUE) != 0)
+                                       goto errout;
+                       }
+               }
                if (zpool_create(g_zfs, poolname,
                    nvroot, props, fsprops) == 0) {
                        zfs_handle_t *pool = zfs_open(g_zfs, poolname,
@@ -1148,6 +1230,10 @@ print_status_config(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
                        (void) printf(gettext("newer version"));
                        break;
 
+               case VDEV_AUX_UNSUP_FEAT:
+                       (void) printf(gettext("unsupported feature(s)"));
+                       break;
+
                case VDEV_AUX_SPARED:
                        verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
                            &cb.cb_guid) == 0);
@@ -1265,6 +1351,10 @@ print_import_config(const char *name, nvlist_t *nv, int namewidth, int depth)
                        (void) printf(gettext("newer version"));
                        break;
 
+               case VDEV_AUX_UNSUP_FEAT:
+                       (void) printf(gettext("unsupported feature(s)"));
+                       break;
+
                case VDEV_AUX_ERR_EXCEEDED:
                        (void) printf(gettext("too many errors"));
                        break;
@@ -1422,8 +1512,8 @@ show_import(nvlist_t *config)
                break;
 
        case ZPOOL_STATUS_VERSION_OLDER:
-               (void) printf(gettext(" status: The pool is formatted using an "
-                   "older on-disk version.\n"));
+               (void) printf(gettext(" status: The pool is formatted using a "
+                   "legacy on-disk version.\n"));
                break;
 
        case ZPOOL_STATUS_VERSION_NEWER:
@@ -1431,6 +1521,25 @@ show_import(nvlist_t *config)
                    "incompatible version.\n"));
                break;
 
+       case ZPOOL_STATUS_FEAT_DISABLED:
+               (void) printf(gettext(" status: Some supported features are "
+                   "not enabled on the pool.\n"));
+               break;
+
+       case ZPOOL_STATUS_UNSUP_FEAT_READ:
+               (void) printf(gettext("status: The pool uses the following "
+                   "feature(s) not supported on this sytem:\n"));
+               zpool_print_unsup_feat(config);
+               break;
+
+       case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
+               (void) printf(gettext("status: The pool can only be accessed "
+                   "in read-only mode on this system. It\n\tcannot be "
+                   "accessed in read-write mode because it uses the "
+                   "following\n\tfeature(s) not supported on this system:\n"));
+               zpool_print_unsup_feat(config);
+               break;
+
        case ZPOOL_STATUS_HOSTID_MISMATCH:
                (void) printf(gettext(" status: The pool was last accessed by "
                    "another system.\n"));
@@ -1463,19 +1572,21 @@ show_import(nvlist_t *config)
         * Print out an action according to the overall state of the pool.
         */
        if (vs->vs_state == VDEV_STATE_HEALTHY) {
-               if (reason == ZPOOL_STATUS_VERSION_OLDER)
+               if (reason == ZPOOL_STATUS_VERSION_OLDER ||
+                   reason == ZPOOL_STATUS_FEAT_DISABLED) {
                        (void) printf(gettext(" action: The pool can be "
                            "imported using its name or numeric identifier, "
                            "though\n\tsome features will not be available "
                            "without an explicit 'zpool upgrade'.\n"));
-               else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH)
+               } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
                        (void) printf(gettext(" action: The pool can be "
                            "imported using its name or numeric "
                            "identifier and\n\tthe '-f' flag.\n"));
-               else
+               } else {
                        (void) printf(gettext(" action: The pool can be "
                            "imported using its name or numeric "
                            "identifier.\n"));
+               }
        } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
                (void) printf(gettext(" action: The pool can be imported "
                    "despite missing or damaged devices.  The\n\tfault "
@@ -1488,6 +1599,20 @@ show_import(nvlist_t *config)
                            "newer\n\tsoftware, or recreate the pool from "
                            "backup.\n"));
                        break;
+               case ZPOOL_STATUS_UNSUP_FEAT_READ:
+                       (void) printf(gettext("action: The pool cannot be "
+                           "imported. Access the pool on a system that "
+                           "supports\n\tthe required feature(s), or recreate "
+                           "the pool from backup.\n"));
+                       break;
+               case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
+                       (void) printf(gettext("action: The pool cannot be "
+                           "imported in read-write mode. Import the pool "
+                           "with\n"
+                           "\t\"-o readonly=on\", access the pool on a system "
+                           "that supports the\n\trequired feature(s), or "
+                           "recreate the pool from backup.\n"));
+                       break;
                case ZPOOL_STATUS_MISSING_DEV_R:
                case ZPOOL_STATUS_MISSING_DEV_NR:
                case ZPOOL_STATUS_BAD_GUID_SUM:
@@ -1563,9 +1688,9 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
            ZPOOL_CONFIG_POOL_STATE, &state) == 0);
        verify(nvlist_lookup_uint64(config,
            ZPOOL_CONFIG_VERSION, &version) == 0);
-       if (version > SPA_VERSION) {
+       if (!SPA_VERSION_IS_SUPPORTED(version)) {
                (void) fprintf(stderr, gettext("cannot import '%s': pool "
-                   "is formatted using a newer ZFS version\n"), name);
+                   "is formatted using an unsupported ZFS version\n"), name);
                return (1);
        } else if (state != POOL_STATE_EXPORTED &&
            !(flags & ZFS_IMPORT_ANY_HOST)) {
@@ -2556,15 +2681,13 @@ static void
 print_header(list_cbdata_t *cb)
 {
        zprop_list_t *pl = cb->cb_proplist;
+       char headerbuf[ZPOOL_MAXPROPLEN];
        const char *header;
        boolean_t first = B_TRUE;
        boolean_t right_justify;
        size_t width = 0;
 
        for (; pl != NULL; pl = pl->pl_next) {
-               if (pl->pl_prop == ZPROP_INVAL)
-                       continue;
-
                width = pl->pl_width;
                if (first && cb->cb_verbose) {
                        /*
@@ -2579,8 +2702,18 @@ print_header(list_cbdata_t *cb)
                else
                        first = B_FALSE;
 
-               header = zpool_prop_column_name(pl->pl_prop);
-               right_justify = zpool_prop_align_right(pl->pl_prop);
+               right_justify = B_FALSE;
+               if (pl->pl_prop != ZPROP_INVAL) {
+                       header = zpool_prop_column_name(pl->pl_prop);
+                       right_justify = zpool_prop_align_right(pl->pl_prop);
+               } else {
+                       int i;
+
+                       for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
+                               headerbuf[i] = toupper(pl->pl_user_prop[i]);
+                       headerbuf[i] = '\0';
+                       header = headerbuf;
+               }
 
                if (pl->pl_next == NULL && !right_justify)
                        (void) printf("%s", header);
@@ -2639,6 +2772,11 @@ print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
                                propstr = property;
 
                        right_justify = zpool_prop_align_right(pl->pl_prop);
+               } else if ((zpool_prop_feature(pl->pl_user_prop) ||
+                   zpool_prop_unsupported(pl->pl_user_prop)) &&
+                   zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
+                   sizeof (property)) == 0) {
+                       propstr = property;
                } else {
                        propstr = "-";
                }
@@ -3474,22 +3612,37 @@ zpool_do_reguid(int argc, char **argv)
  * zpool reopen <pool>
  *
  * Reopen the pool so that the kernel can update the sizes of all vdevs.
- *
- * NOTE: This command is currently undocumented.  If the command is ever
- * exposed then the appropriate usage() messages will need to be made.
  */
 int
 zpool_do_reopen(int argc, char **argv)
 {
+       int c;
        int ret = 0;
        zpool_handle_t *zhp;
        char *pool;
 
+       /* check options */
+       while ((c = getopt(argc, argv, "")) != -1) {
+               switch (c) {
+               case '?':
+                       (void) fprintf(stderr, gettext("invalid option '%c'\n"),
+                           optopt);
+                       usage(B_FALSE);
+               }
+       }
+
        argc--;
        argv++;
 
-       if (argc != 1)
-               return (2);
+       if (argc < 1) {
+               (void) fprintf(stderr, gettext("missing pool name\n"));
+               usage(B_FALSE);
+       }
+
+       if (argc > 1) {
+               (void) fprintf(stderr, gettext("too many arguments\n"));
+               usage(B_FALSE);
+       }
 
        pool = argv[0];
        if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL)
@@ -3587,7 +3740,7 @@ print_scan_status(pool_scan_stat_t *ps)
        double fraction_done;
        char processed_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
 
-       (void) printf(gettext(" scan: "));
+       (void) printf(gettext("  scan: "));
 
        /* If there's never been a scan, there's not much to say. */
        if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
@@ -3775,10 +3928,16 @@ print_dedup_stats(nvlist_t *config)
         * table continue processing the stats.
         */
        if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
-           (uint64_t **)&ddo, &c) != 0 || ddo->ddo_count == 0)
+           (uint64_t **)&ddo, &c) != 0)
                return;
 
        (void) printf("\n");
+       (void) printf(gettext(" dedup: "));
+       if (ddo->ddo_count == 0) {
+               (void) printf(gettext("no DDT entries\n"));
+               return;
+       }
+
        (void) printf("DDT entries %llu, size %llu on disk, %llu in core\n",
            (u_longlong_t)ddo->ddo_count,
            (u_longlong_t)ddo->ddo_dspace,
@@ -3826,7 +3985,10 @@ status_callback(zpool_handle_t *zhp, void *data)
         * If we were given 'zpool status -x', only report those pools with
         * problems.
         */
-       if (reason == ZPOOL_STATUS_OK && cbp->cb_explain) {
+       if (cbp->cb_explain &&
+           (reason == ZPOOL_STATUS_OK ||
+           reason == ZPOOL_STATUS_VERSION_OLDER ||
+           reason == ZPOOL_STATUS_FEAT_DISABLED)) {
                if (!cbp->cb_allpools) {
                        (void) printf(gettext("pool '%s' is healthy\n"),
                            zpool_get_name(zhp));
@@ -3941,12 +4103,13 @@ status_callback(zpool_handle_t *zhp, void *data)
                break;
 
        case ZPOOL_STATUS_VERSION_OLDER:
-               (void) printf(gettext("status: The pool is formatted using an "
-                   "older on-disk format.  The pool can\n\tstill be used, but "
-                   "some features are unavailable.\n"));
+               (void) printf(gettext("status: The pool is formatted using a "
+                   "legacy on-disk format.  The pool can\n\tstill be used, "
+                   "but some features are unavailable.\n"));
                (void) printf(gettext("action: Upgrade the pool using 'zpool "
                    "upgrade'.  Once this is done, the\n\tpool will no longer "
-                   "be accessible on older software versions.\n"));
+                   "be accessible on software that does not support\n\t"
+                   "feature flags.\n"));
                break;
 
        case ZPOOL_STATUS_VERSION_NEWER:
@@ -3958,6 +4121,41 @@ status_callback(zpool_handle_t *zhp, void *data)
                    "backup.\n"));
                break;
 
+       case ZPOOL_STATUS_FEAT_DISABLED:
+               (void) printf(gettext("status: Some supported features are not "
+                   "enabled on the pool. The pool can\n\tstill be used, but "
+                   "some features are unavailable.\n"));
+               (void) printf(gettext("action: Enable all features using "
+                   "'zpool upgrade'. Once this is done,\n\tthe pool may no "
+                   "longer be accessible by software that does not support\n\t"
+                   "the features. See zpool-features(5) for details.\n"));
+               break;
+
+       case ZPOOL_STATUS_UNSUP_FEAT_READ:
+               (void) printf(gettext("status: The pool cannot be accessed on "
+                   "this system because it uses the\n\tfollowing feature(s) "
+                   "not supported on this system:\n"));
+               zpool_print_unsup_feat(config);
+               (void) printf("\n");
+               (void) printf(gettext("action: Access the pool from a system "
+                   "that supports the required feature(s),\n\tor restore the "
+                   "pool from backup.\n"));
+               break;
+
+       case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
+               (void) printf(gettext("status: The pool can only be accessed "
+                   "in read-only mode on this system. It\n\tcannot be "
+                   "accessed in read-write mode because it uses the "
+                   "following\n\tfeature(s) not supported on this system:\n"));
+               zpool_print_unsup_feat(config);
+               (void) printf("\n");
+               (void) printf(gettext("action: The pool cannot be accessed in "
+                   "read-write mode. Import the pool with\n"
+                   "\t\"-o readonly=on\", access the pool from a system that "
+                   "supports the\n\trequired feature(s), or restore the "
+                   "pool from backup.\n"));
+               break;
+
        case ZPOOL_STATUS_FAULTED_DEV_R:
                (void) printf(gettext("status: One or more devices are "
                    "faulted in response to persistent errors.\n\tSufficient "
@@ -4162,56 +4360,162 @@ zpool_do_status(int argc, char **argv)
 }
 
 typedef struct upgrade_cbdata {
-       int     cb_all;
        int     cb_first;
-       int     cb_newer;
        int     cb_argc;
        uint64_t cb_version;
        char    **cb_argv;
 } upgrade_cbdata_t;
 
 static int
+upgrade_version(zpool_handle_t *zhp, uint64_t version)
+{
+       int ret;
+       nvlist_t *config;
+       uint64_t oldversion;
+
+       config = zpool_get_config(zhp, NULL);
+       verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+           &oldversion) == 0);
+
+       assert(SPA_VERSION_IS_SUPPORTED(oldversion));
+       assert(oldversion < version);
+
+       ret = zpool_upgrade(zhp, version);
+       if (ret != 0)
+               return (ret);
+
+       if (version >= SPA_VERSION_FEATURES) {
+               (void) printf(gettext("Successfully upgraded "
+                   "'%s' from version %llu to feature flags.\n"),
+                   zpool_get_name(zhp), (u_longlong_t) oldversion);
+       } else {
+               (void) printf(gettext("Successfully upgraded "
+                   "'%s' from version %llu to version %llu.\n"),
+                   zpool_get_name(zhp), (u_longlong_t) oldversion,
+                   (u_longlong_t) version);
+       }
+
+       return (0);
+}
+
+static int
+upgrade_enable_all(zpool_handle_t *zhp, int *countp)
+{
+       int i, ret, count;
+       boolean_t firstff = B_TRUE;
+       nvlist_t *enabled = zpool_get_features(zhp);
+
+       count = 0;
+       for (i = 0; i < SPA_FEATURES; i++) {
+               const char *fname = spa_feature_table[i].fi_uname;
+               const char *fguid = spa_feature_table[i].fi_guid;
+               if (!nvlist_exists(enabled, fguid)) {
+                       char *propname;
+                       verify(-1 != asprintf(&propname, "feature@%s", fname));
+                       ret = zpool_set_prop(zhp, propname,
+                           ZFS_FEATURE_ENABLED);
+                       if (ret != 0) {
+                               free(propname);
+                               return (ret);
+                       }
+                       count++;
+
+                       if (firstff) {
+                               (void) printf(gettext("Enabled the "
+                                   "following features on '%s':\n"),
+                                   zpool_get_name(zhp));
+                               firstff = B_FALSE;
+                       }
+                       (void) printf(gettext("  %s\n"), fname);
+                       free(propname);
+               }
+       }
+
+       if (countp != NULL)
+               *countp = count;
+       return (0);
+}
+
+static int
 upgrade_cb(zpool_handle_t *zhp, void *arg)
 {
        upgrade_cbdata_t *cbp = arg;
        nvlist_t *config;
        uint64_t version;
-       int ret = 0;
+       boolean_t printnl = B_FALSE;
+       int ret;
 
        config = zpool_get_config(zhp, NULL);
        verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
            &version) == 0);
 
-       if (!cbp->cb_newer && version < SPA_VERSION) {
-               if (!cbp->cb_all) {
-                       if (cbp->cb_first) {
-                               (void) printf(gettext("The following pools are "
-                                   "out of date, and can be upgraded.  After "
-                                   "being\nupgraded, these pools will no "
-                                   "longer be accessible by older software "
-                                   "versions.\n\n"));
-                               (void) printf(gettext("VER  POOL\n"));
-                               (void) printf(gettext("---  ------------\n"));
-                               cbp->cb_first = B_FALSE;
-                       }
+       assert(SPA_VERSION_IS_SUPPORTED(version));
 
-                       (void) printf("%2llu   %s\n", (u_longlong_t)version,
-                           zpool_get_name(zhp));
-               } else {
+       if (version < cbp->cb_version) {
+               cbp->cb_first = B_FALSE;
+               ret = upgrade_version(zhp, cbp->cb_version);
+               if (ret != 0)
+                       return (ret);
+               printnl = B_TRUE;
+
+#if 0
+               /*
+                * XXX: This code can be enabled when Illumos commit
+                * 4445fffbbb1ea25fd0e9ea68b9380dd7a6709025 is merged.
+                * It reworks the history logging among other things.
+                */
+
+               /*
+                * If they did "zpool upgrade -a", then we could
+                * be doing ioctls to different pools.  We need
+                * to log this history once to each pool, and bypass
+                * the normal history logging that happens in main().
+                */
+               (void) zpool_log_history(g_zfs, history_str);
+               log_history = B_FALSE;
+#endif
+       }
+
+       if (cbp->cb_version >= SPA_VERSION_FEATURES) {
+               int count;
+               ret = upgrade_enable_all(zhp, &count);
+               if (ret != 0)
+                       return (ret);
+
+               if (count > 0) {
                        cbp->cb_first = B_FALSE;
-                       ret = zpool_upgrade(zhp, cbp->cb_version);
-                       if (!ret) {
-                               (void) printf(gettext("Successfully upgraded "
-                                   "'%s'\n\n"), zpool_get_name(zhp));
-                       }
+                       printnl = B_TRUE;
                }
-       } else if (cbp->cb_newer && version > SPA_VERSION) {
-               assert(!cbp->cb_all);
+       }
 
+       if (printnl) {
+               (void) printf(gettext("\n"));
+       }
+
+       return (0);
+}
+
+static int
+upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
+{
+       upgrade_cbdata_t *cbp = arg;
+       nvlist_t *config;
+       uint64_t version;
+
+       config = zpool_get_config(zhp, NULL);
+       verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+           &version) == 0);
+
+       assert(SPA_VERSION_IS_SUPPORTED(version));
+
+       if (version < SPA_VERSION_FEATURES) {
                if (cbp->cb_first) {
                        (void) printf(gettext("The following pools are "
-                           "formatted using a newer software version and\n"
-                           "cannot be accessed on the current system.\n\n"));
+                           "formatted with legacy version numbers and can\n"
+                           "be upgraded to use feature flags.  After "
+                           "being upgraded, these pools\nwill no "
+                           "longer be accessible by software that does not "
+                           "support feature\nflags.\n\n"));
                        (void) printf(gettext("VER  POOL\n"));
                        (void) printf(gettext("---  ------------\n"));
                        cbp->cb_first = B_FALSE;
@@ -4221,14 +4525,65 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
                    zpool_get_name(zhp));
        }
 
-       zpool_close(zhp);
-       return (ret);
+       return (0);
+}
+
+static int
+upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
+{
+       upgrade_cbdata_t *cbp = arg;
+       nvlist_t *config;
+       uint64_t version;
+
+       config = zpool_get_config(zhp, NULL);
+       verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+           &version) == 0);
+
+       if (version >= SPA_VERSION_FEATURES) {
+               int i;
+               boolean_t poolfirst = B_TRUE;
+               nvlist_t *enabled = zpool_get_features(zhp);
+
+               for (i = 0; i < SPA_FEATURES; i++) {
+                       const char *fguid = spa_feature_table[i].fi_guid;
+                       const char *fname = spa_feature_table[i].fi_uname;
+                       if (!nvlist_exists(enabled, fguid)) {
+                               if (cbp->cb_first) {
+                                       (void) printf(gettext("\nSome "
+                                           "supported features are not "
+                                           "enabled on the following pools. "
+                                           "Once a\nfeature is enabled the "
+                                           "pool may become incompatible with "
+                                           "software\nthat does not support "
+                                           "the feature. See "
+                                           "zpool-features(5) for "
+                                           "details.\n\n"));
+                                       (void) printf(gettext("POOL  "
+                                           "FEATURE\n"));
+                                       (void) printf(gettext("------"
+                                           "---------\n"));
+                                       cbp->cb_first = B_FALSE;
+                               }
+
+                               if (poolfirst) {
+                                       (void) printf(gettext("%s\n"),
+                                           zpool_get_name(zhp));
+                                       poolfirst = B_FALSE;
+                               }
+
+                               (void) printf(gettext("      %s\n"), fname);
+                       }
+               }
+       }
+
+       return (0);
 }
 
 /* ARGSUSED */
 static int
 upgrade_one(zpool_handle_t *zhp, void *data)
 {
+       boolean_t printnl = B_FALSE;
        upgrade_cbdata_t *cbp = data;
        uint64_t cur_version;
        int ret;
@@ -4243,26 +4598,45 @@ upgrade_one(zpool_handle_t *zhp, void *data)
        cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
        if (cur_version > cbp->cb_version) {
                (void) printf(gettext("Pool '%s' is already formatted "
-                   "using more current version '%llu'.\n"),
+                   "using more current version '%llu'.\n\n"),
                    zpool_get_name(zhp), (u_longlong_t) cur_version);
                return (0);
        }
-       if (cur_version == cbp->cb_version) {
+
+       if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
                (void) printf(gettext("Pool '%s' is already formatted "
-                   "using the current version.\n"), zpool_get_name(zhp));
+                   "using version %llu.\n\n"), zpool_get_name(zhp),
+                   (u_longlong_t) cbp->cb_version);
                return (0);
        }
 
-       ret = zpool_upgrade(zhp, cbp->cb_version);
+       if (cur_version != cbp->cb_version) {
+               printnl = B_TRUE;
+               ret = upgrade_version(zhp, cbp->cb_version);
+               if (ret != 0)
+                       return (ret);
+       }
+
+       if (cbp->cb_version >= SPA_VERSION_FEATURES) {
+               int count = 0;
+               ret = upgrade_enable_all(zhp, &count);
+               if (ret != 0)
+                       return (ret);
+
+               if (count != 0) {
+                       printnl = B_TRUE;
+               } else if (cur_version == SPA_VERSION) {
+                       (void) printf(gettext("Pool '%s' already has all "
+                           "supported features enabled.\n"),
+                           zpool_get_name(zhp));
+               }
+       }
 
-       if (!ret) {
-               (void) printf(gettext("Successfully upgraded '%s' "
-                   "from version %llu to version %llu\n\n"),
-                   zpool_get_name(zhp), (u_longlong_t)cur_version,
-                   (u_longlong_t)cbp->cb_version);
+       if (printnl) {
+               (void) printf(gettext("\n"));
        }
 
-       return (ret != 0);
+       return (0);
 }
 
 /*
@@ -4281,6 +4655,7 @@ zpool_do_upgrade(int argc, char **argv)
        upgrade_cbdata_t cb = { 0 };
        int ret = 0;
        boolean_t showversions = B_FALSE;
+       boolean_t upgradeall = B_FALSE;
        char *end;
 
 
@@ -4288,15 +4663,15 @@ zpool_do_upgrade(int argc, char **argv)
        while ((c = getopt(argc, argv, ":avV:")) != -1) {
                switch (c) {
                case 'a':
-                       cb.cb_all = B_TRUE;
+                       upgradeall = B_TRUE;
                        break;
                case 'v':
                        showversions = B_TRUE;
                        break;
                case 'V':
                        cb.cb_version = strtoll(optarg, &end, 10);
-                       if (*end != '\0' || cb.cb_version > SPA_VERSION ||
-                           cb.cb_version < SPA_VERSION_1) {
+                       if (*end != '\0' ||
+                           !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
                                (void) fprintf(stderr,
                                    gettext("invalid version '%s'\n"), optarg);
                                usage(B_FALSE);
@@ -4321,19 +4696,19 @@ zpool_do_upgrade(int argc, char **argv)
 
        if (cb.cb_version == 0) {
                cb.cb_version = SPA_VERSION;
-       } else if (!cb.cb_all && argc == 0) {
+       } else if (!upgradeall && argc == 0) {
                (void) fprintf(stderr, gettext("-V option is "
                    "incompatible with other arguments\n"));
                usage(B_FALSE);
        }
 
        if (showversions) {
-               if (cb.cb_all || argc != 0) {
+               if (upgradeall || argc != 0) {
                        (void) fprintf(stderr, gettext("-v option is "
                            "incompatible with other arguments\n"));
                        usage(B_FALSE);
                }
-       } else if (cb.cb_all) {
+       } else if (upgradeall) {
                if (argc != 0) {
                        (void) fprintf(stderr, gettext("-a option should not "
                            "be used along with a pool name\n"));
@@ -4341,11 +4716,27 @@ zpool_do_upgrade(int argc, char **argv)
                }
        }
 
-       (void) printf(gettext("This system is currently running "
-           "ZFS pool version %llu.\n\n"), SPA_VERSION);
-       cb.cb_first = B_TRUE;
+       (void) printf(gettext("This system supports ZFS pool feature "
+           "flags.\n\n"));
        if (showversions) {
-               (void) printf(gettext("The following versions are "
+               int i;
+
+               (void) printf(gettext("The following features are "
+                   "supported:\n\n"));
+               (void) printf(gettext("FEAT DESCRIPTION\n"));
+               (void) printf("----------------------------------------------"
+                   "---------------\n");
+               for (i = 0; i < SPA_FEATURES; i++) {
+                       zfeature_info_t *fi = &spa_feature_table[i];
+                       const char *ro = fi->fi_can_readonly ?
+                           " (read-only compatible)" : "";
+
+                       (void) printf("%-37s%s\n", fi->fi_uname, ro);
+                       (void) printf("     %s\n", fi->fi_desc);
+               }
+               (void) printf("\n");
+
+               (void) printf(gettext("The following legacy versions are also "
                    "supported:\n\n"));
                (void) printf(gettext("VER  DESCRIPTION\n"));
                (void) printf("---  -----------------------------------------"
@@ -4388,32 +4779,44 @@ zpool_do_upgrade(int argc, char **argv)
                (void) printf(gettext("\nFor more information on a particular "
                    "version, including supported releases,\n"));
                (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
-       } else if (argc == 0) {
-               int notfound;
-
+       } else if (argc == 0 && upgradeall) {
+               cb.cb_first = B_TRUE;
                ret = zpool_iter(g_zfs, upgrade_cb, &cb);
-               notfound = cb.cb_first;
-
-               if (!cb.cb_all && ret == 0) {
-                       if (!cb.cb_first)
-                               (void) printf("\n");
-                       cb.cb_first = B_TRUE;
-                       cb.cb_newer = B_TRUE;
-                       ret = zpool_iter(g_zfs, upgrade_cb, &cb);
-                       if (!cb.cb_first) {
-                               notfound = B_FALSE;
-                               (void) printf("\n");
+               if (ret == 0 && cb.cb_first) {
+                       if (cb.cb_version == SPA_VERSION) {
+                               (void) printf(gettext("All pools are already "
+                                   "formatted using feature flags.\n\n"));
+                               (void) printf(gettext("Every feature flags "
+                                   "pool already has all supported features "
+                                   "enabled.\n"));
+                       } else {
+                               (void) printf(gettext("All pools are already "
+                                   "formatted with version %llu or higher.\n"),
+                                   (u_longlong_t) cb.cb_version);
                        }
                }
+       } else if (argc == 0) {
+               cb.cb_first = B_TRUE;
+               ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
+               assert(ret == 0);
+
+               if (cb.cb_first) {
+                       (void) printf(gettext("All pools are formatted "
+                           "using feature flags.\n\n"));
+               } else {
+                       (void) printf(gettext("\nUse 'zpool upgrade -v' "
+                           "for a list of available legacy versions.\n"));
+               }
 
-               if (ret == 0) {
-                       if (notfound)
-                               (void) printf(gettext("All pools are formatted "
-                                   "using this version.\n"));
-                       else if (!cb.cb_all)
-                               (void) printf(gettext("Use 'zpool upgrade -v' "
-                                   "for a list of available versions and "
-                                   "their associated\nfeatures.\n"));
+               cb.cb_first = B_TRUE;
+               ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
+               assert(ret == 0);
+
+               if (cb.cb_first) {
+                       (void) printf(gettext("Every feature flags pool has "
+                           "all supported features enabled.\n"));
+               } else {
+                       (void) printf(gettext("\n"));
                }
        } else {
                ret = for_each_pool(argc, argv, B_FALSE, NULL,
@@ -4840,6 +5243,7 @@ zpool_do_events_next(ev_opts_t *opts)
                        zpool_do_events_nvprint(nvl, 8);
                        printf(gettext("\n"));
                }
+               (void) fflush(stdout);
 
                nvlist_free(nvl);
        }
@@ -4923,13 +5327,26 @@ get_callback(zpool_handle_t *zhp, void *data)
                    pl == cbp->cb_proplist)
                        continue;
 
-               if (zpool_get_prop(zhp, pl->pl_prop,
-                   value, sizeof (value), &srctype) != 0)
-                       continue;
+               if (pl->pl_prop == ZPROP_INVAL &&
+                   (zpool_prop_feature(pl->pl_user_prop) ||
+                   zpool_prop_unsupported(pl->pl_user_prop))) {
+                       srctype = ZPROP_SRC_LOCAL;
+
+                       if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
+                           value, sizeof (value)) == 0) {
+                               zprop_print_one_property(zpool_get_name(zhp),
+                                   cbp, pl->pl_user_prop, value, srctype,
+                                   NULL, NULL);
+                       }
+               } else {
+                       if (zpool_get_prop(zhp, pl->pl_prop, value,
+                           sizeof (value), &srctype) != 0)
+                               continue;
 
-               zprop_print_one_property(zpool_get_name(zhp), cbp,
-                   zpool_prop_to_name(pl->pl_prop), value, srctype, NULL,
-                   NULL);
+                       zprop_print_one_property(zpool_get_name(zhp), cbp,
+                           zpool_prop_to_name(pl->pl_prop), value, srctype,
+                           NULL, NULL);
+               }
        }
        return (0);
 }
@@ -4941,8 +5358,11 @@ zpool_do_get(int argc, char **argv)
        zprop_list_t fake_name = { 0 };
        int ret;
 
-       if (argc < 3)
+       if (argc < 2) {
+               (void) fprintf(stderr, gettext("missing property "
+                   "argument\n"));
                usage(B_FALSE);
+       }
 
        cb.cb_first = B_TRUE;
        cb.cb_sources = ZPROP_SRC_ALL;
@@ -4952,7 +5372,7 @@ zpool_do_get(int argc, char **argv)
        cb.cb_columns[3] = GET_COL_SOURCE;
        cb.cb_type = ZFS_TYPE_POOL;
 
-       if (zprop_get_list(g_zfs, argv[1],  &cb.cb_proplist,
+       if (zprop_get_list(g_zfs, argv[1], &cb.cb_proplist,
            ZFS_TYPE_POOL) != 0)
                usage(B_FALSE);