Linux patch 6.1.506.1-56

Signed-off-by: Mike Pagano <mpagano@gentoo.org>

2 files changed, 31695 insertions, 0 deletions

diff --git a/0000_README b/0000_README
index 6ce2c13b..870654fc 100644
--- a/0000_README
+++ b/0000_README
@@ -239,6 +239,10 @@ Patch:  1048_linux-6.1.49.patch
 From:   https://www.kernel.org
 Desc:   Linux 6.1.49
 
+Patch:  1049_linux-6.1.50.patch
+From:   https://www.kernel.org
+Desc:   Linux 6.1.50
+
 Patch:  1500_XATTR_USER_PREFIX.patch
 From:   https://bugs.gentoo.org/show_bug.cgi?id=470644
 Desc:   Support for namespace user.pax.* on tmpfs.
diff --git a/1049_linux-6.1.50.patch b/1049_linux-6.1.50.patch
new file mode 100644
index 00000000..4a052c91
--- /dev/null
+++ b/1049_linux-6.1.50.patch
@@ -0,0 +1,31691 @@
+diff --git a/MAINTAINERS b/MAINTAINERS
+index 379387e20a96d..07a9c274c0e29 100644
+--- a/MAINTAINERS
++++ b/MAINTAINERS
+@@ -6027,7 +6027,7 @@ S:	Supported
+ F:	Documentation/networking/devlink
+ F:	include/net/devlink.h
+ F:	include/uapi/linux/devlink.h
+-F:	net/core/devlink.c
++F:	net/devlink/
+ 
+ DH ELECTRONICS IMX6 DHCOM BOARD SUPPORT
+ M:	Christoph Niedermaier <cniedermaier@dh-electronics.com>
+diff --git a/Makefile b/Makefile
+index 61ebd54aba899..e5e1fdeef8bf0 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,7 +1,7 @@
+ # SPDX-License-Identifier: GPL-2.0
+ VERSION = 6
+ PATCHLEVEL = 1
+-SUBLEVEL = 49
++SUBLEVEL = 50
+ EXTRAVERSION =
+ NAME = Curry Ramen
+ 
+diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h
+index c0983130a44c9..e0a4da4cfd8bc 100644
+--- a/arch/mips/include/asm/cpu-features.h
++++ b/arch/mips/include/asm/cpu-features.h
+@@ -121,7 +121,24 @@
+ #define cpu_has_4k_cache	__isa_ge_or_opt(1, MIPS_CPU_4K_CACHE)
+ #endif
+ #ifndef cpu_has_octeon_cache
+-#define cpu_has_octeon_cache	0
++#define cpu_has_octeon_cache						\
++({									\
++	int __res;							\
++									\
++	switch (boot_cpu_type()) {					\
++	case CPU_CAVIUM_OCTEON:						\
++	case CPU_CAVIUM_OCTEON_PLUS:					\
++	case CPU_CAVIUM_OCTEON2:					\
++	case CPU_CAVIUM_OCTEON3:					\
++		__res = 1;						\
++		break;							\
++									\
++	default:							\
++		__res = 0;						\
++	}								\
++									\
++	__res;								\
++})
+ #endif
+ /* Don't override `cpu_has_fpu' to 1 or the "nofpu" option won't work.  */
+ #ifndef cpu_has_fpu
+@@ -351,7 +368,7 @@
+ ({									\
+ 	int __res;							\
+ 									\
+-	switch (current_cpu_type()) {					\
++	switch (boot_cpu_type()) {					\
+ 	case CPU_M14KC:							\
+ 	case CPU_74K:							\
+ 	case CPU_1074K:							\
+diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
+index 6bf8dc0b8f935..d702359f8ab5e 100644
+--- a/arch/riscv/Kconfig
++++ b/arch/riscv/Kconfig
+@@ -447,24 +447,30 @@ config TOOLCHAIN_HAS_ZIHINTPAUSE
+ config TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI
+ 	def_bool y
+ 	# https://sourceware.org/git/?p=binutils-gdb.git;a=commit;h=aed44286efa8ae8717a77d94b51ac3614e2ca6dc
+-	depends on AS_IS_GNU && AS_VERSION >= 23800
++	# https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=98416dbb0a62579d4a7a4a76bab51b5b52fec2cd
++	depends on AS_IS_GNU && AS_VERSION >= 23600
+ 	help
+-	  Newer binutils versions default to ISA spec version 20191213 which
+-	  moves some instructions from the I extension to the Zicsr and Zifencei
+-	  extensions.
++	  Binutils-2.38 and GCC-12.1.0 bumped the default ISA spec to the newer
++	  20191213 version, which moves some instructions from the I extension to
++	  the Zicsr and Zifencei extensions. This requires explicitly specifying
++	  Zicsr and Zifencei when binutils >= 2.38 or GCC >= 12.1.0. Zicsr
++	  and Zifencei are supported in binutils from version 2.36 onwards.
++	  To make life easier, and avoid forcing toolchains that default to a
++	  newer ISA spec to version 2.2, relax the check to binutils >= 2.36.
++	  For clang < 17 or GCC < 11.3.0, for which this is not possible or need
++	  special treatment, this is dealt with in TOOLCHAIN_NEEDS_OLD_ISA_SPEC.
+ 
+ config TOOLCHAIN_NEEDS_OLD_ISA_SPEC
+ 	def_bool y
+ 	depends on TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI
+ 	# https://github.com/llvm/llvm-project/commit/22e199e6afb1263c943c0c0d4498694e15bf8a16
+-	depends on CC_IS_CLANG && CLANG_VERSION < 170000
++	# https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=d29f5d6ab513c52fd872f532c492e35ae9fd6671
++	depends on (CC_IS_CLANG && CLANG_VERSION < 170000) || (CC_IS_GCC && GCC_VERSION < 110300)
+ 	help
+-	  Certain versions of clang do not support zicsr and zifencei via -march
+-	  but newer versions of binutils require it for the reasons noted in the
+-	  help text of CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI. This
+-	  option causes an older ISA spec compatible with these older versions
+-	  of clang to be passed to GAS, which has the same result as passing zicsr
+-	  and zifencei to -march.
++	  Certain versions of clang and GCC do not support zicsr and zifencei via
++	  -march. This option causes an older ISA spec compatible with these older
++	  versions of clang and GCC to be passed to GAS, which has the same result
++	  as passing zicsr and zifencei to -march.
+ 
+ config FPU
+ 	bool "FPU support"
+diff --git a/arch/riscv/kernel/compat_vdso/Makefile b/arch/riscv/kernel/compat_vdso/Makefile
+index 7f34f3c7c8827..737c0857b14cd 100644
+--- a/arch/riscv/kernel/compat_vdso/Makefile
++++ b/arch/riscv/kernel/compat_vdso/Makefile
+@@ -11,7 +11,13 @@ compat_vdso-syms += flush_icache
+ COMPAT_CC := $(CC)
+ COMPAT_LD := $(LD)
+ 
+-COMPAT_CC_FLAGS := -march=rv32g -mabi=ilp32
++# binutils 2.35 does not support the zifencei extension, but in the ISA
++# spec 20191213, G stands for IMAFD_ZICSR_ZIFENCEI.
++ifdef CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI
++	COMPAT_CC_FLAGS := -march=rv32g -mabi=ilp32
++else
++	COMPAT_CC_FLAGS := -march=rv32imafd -mabi=ilp32
++endif
+ COMPAT_LD_FLAGS := -melf32lriscv
+ 
+ # Disable attributes, as they're useless and break the build.
+diff --git a/arch/x86/kernel/fpu/context.h b/arch/x86/kernel/fpu/context.h
+index 9fcfa5c4dad79..71b5059e092ab 100644
+--- a/arch/x86/kernel/fpu/context.h
++++ b/arch/x86/kernel/fpu/context.h
+@@ -19,8 +19,7 @@
+  * FPU state for a task MUST let the rest of the kernel know that the
+  * FPU registers are no longer valid for this task.
+  *
+- * Either one of these invalidation functions is enough. Invalidate
+- * a resource you control: CPU if using the CPU for something else
++ * Invalidate a resource you control: CPU if using the CPU for something else
+  * (with preemption disabled), FPU for the current task, or a task that
+  * is prevented from running by the current task.
+  */
+diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
+index caf33486dc5ee..a083f9ac9e4f6 100644
+--- a/arch/x86/kernel/fpu/core.c
++++ b/arch/x86/kernel/fpu/core.c
+@@ -679,7 +679,7 @@ static void fpu_reset_fpregs(void)
+ 	struct fpu *fpu = &current->thread.fpu;
+ 
+ 	fpregs_lock();
+-	fpu__drop(fpu);
++	__fpu_invalidate_fpregs_state(fpu);
+ 	/*
+ 	 * This does not change the actual hardware registers. It just
+ 	 * resets the memory image and sets TIF_NEED_FPU_LOAD so a
+diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
+index 0bab497c94369..1afbc4866b100 100644
+--- a/arch/x86/kernel/fpu/xstate.c
++++ b/arch/x86/kernel/fpu/xstate.c
+@@ -882,6 +882,13 @@ void __init fpu__init_system_xstate(unsigned int legacy_size)
+ 		goto out_disable;
+ 	}
+ 
++	/*
++	 * CPU capabilities initialization runs before FPU init. So
++	 * X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely
++	 * functional, set the feature bit so depending code works.
++	 */
++	setup_force_cpu_cap(X86_FEATURE_OSXSAVE);
++
+ 	print_xstate_offset_size();
+ 	pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
+ 		fpu_kernel_cfg.max_features,
+diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
+index 230108a90cf39..beca03556379d 100644
+--- a/arch/x86/kvm/mmu/mmu.c
++++ b/arch/x86/kvm/mmu/mmu.c
+@@ -4212,7 +4212,8 @@ static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
+  * root was invalidated by a memslot update or a relevant mmu_notifier fired.
+  */
+ static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
+-				struct kvm_page_fault *fault, int mmu_seq)
++				struct kvm_page_fault *fault,
++				unsigned long mmu_seq)
+ {
+ 	struct kvm_mmu_page *sp = to_shadow_page(vcpu->arch.mmu->root.hpa);
+ 
+diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
+index 672f0432d7777..70945f00ec412 100644
+--- a/arch/x86/kvm/mmu/tdp_mmu.c
++++ b/arch/x86/kvm/mmu/tdp_mmu.c
+@@ -51,7 +51,17 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm)
+ 	if (!kvm->arch.tdp_mmu_enabled)
+ 		return;
+ 
+-	/* Also waits for any queued work items.  */
++	/*
++	 * Invalidate all roots, which besides the obvious, schedules all roots
++	 * for zapping and thus puts the TDP MMU's reference to each root, i.e.
++	 * ultimately frees all roots.
++	 */
++	kvm_tdp_mmu_invalidate_all_roots(kvm);
++
++	/*
++	 * Destroying a workqueue also first flushes the workqueue, i.e. no
++	 * need to invoke kvm_tdp_mmu_zap_invalidated_roots().
++	 */
+ 	destroy_workqueue(kvm->arch.tdp_mmu_zap_wq);
+ 
+ 	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_pages));
+@@ -127,16 +137,6 @@ static void tdp_mmu_schedule_zap_root(struct kvm *kvm, struct kvm_mmu_page *root
+ 	queue_work(kvm->arch.tdp_mmu_zap_wq, &root->tdp_mmu_async_work);
+ }
+ 
+-static inline bool kvm_tdp_root_mark_invalid(struct kvm_mmu_page *page)
+-{
+-	union kvm_mmu_page_role role = page->role;
+-	role.invalid = true;
+-
+-	/* No need to use cmpxchg, only the invalid bit can change.  */
+-	role.word = xchg(&page->role.word, role.word);
+-	return role.invalid;
+-}
+-
+ void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
+ 			  bool shared)
+ {
+@@ -145,45 +145,12 @@ void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
+ 	if (!refcount_dec_and_test(&root->tdp_mmu_root_count))
+ 		return;
+ 
+-	WARN_ON(!root->tdp_mmu_page);
+-
+ 	/*
+-	 * The root now has refcount=0.  It is valid, but readers already
+-	 * cannot acquire a reference to it because kvm_tdp_mmu_get_root()
+-	 * rejects it.  This remains true for the rest of the execution
+-	 * of this function, because readers visit valid roots only
+-	 * (except for tdp_mmu_zap_root_work(), which however
+-	 * does not acquire any reference itself).
+-	 *
+-	 * Even though there are flows that need to visit all roots for
+-	 * correctness, they all take mmu_lock for write, so they cannot yet
+-	 * run concurrently. The same is true after kvm_tdp_root_mark_invalid,
+-	 * since the root still has refcount=0.
+-	 *
+-	 * However, tdp_mmu_zap_root can yield, and writers do not expect to
+-	 * see refcount=0 (see for example kvm_tdp_mmu_invalidate_all_roots()).
+-	 * So the root temporarily gets an extra reference, going to refcount=1
+-	 * while staying invalid.  Readers still cannot acquire any reference;
+-	 * but writers are now allowed to run if tdp_mmu_zap_root yields and
+-	 * they might take an extra reference if they themselves yield.
+-	 * Therefore, when the reference is given back by the worker,
+-	 * there is no guarantee that the refcount is still 1.  If not, whoever
+-	 * puts the last reference will free the page, but they will not have to
+-	 * zap the root because a root cannot go from invalid to valid.
++	 * The TDP MMU itself holds a reference to each root until the root is
++	 * explicitly invalidated, i.e. the final reference should be never be
++	 * put for a valid root.
+ 	 */
+-	if (!kvm_tdp_root_mark_invalid(root)) {
+-		refcount_set(&root->tdp_mmu_root_count, 1);
+-
+-		/*
+-		 * Zapping the root in a worker is not just "nice to have";
+-		 * it is required because kvm_tdp_mmu_invalidate_all_roots()
+-		 * skips already-invalid roots.  If kvm_tdp_mmu_put_root() did
+-		 * not add the root to the workqueue, kvm_tdp_mmu_zap_all_fast()
+-		 * might return with some roots not zapped yet.
+-		 */
+-		tdp_mmu_schedule_zap_root(kvm, root);
+-		return;
+-	}
++	KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm);
+ 
+ 	spin_lock(&kvm->arch.tdp_mmu_pages_lock);
+ 	list_del_rcu(&root->link);
+@@ -329,7 +296,14 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu)
+ 	root = tdp_mmu_alloc_sp(vcpu);
+ 	tdp_mmu_init_sp(root, NULL, 0, role);
+ 
+-	refcount_set(&root->tdp_mmu_root_count, 1);
++	/*
++	 * TDP MMU roots are kept until they are explicitly invalidated, either
++	 * by a memslot update or by the destruction of the VM.  Initialize the
++	 * refcount to two; one reference for the vCPU, and one reference for
++	 * the TDP MMU itself, which is held until the root is invalidated and
++	 * is ultimately put by tdp_mmu_zap_root_work().
++	 */
++	refcount_set(&root->tdp_mmu_root_count, 2);
+ 
+ 	spin_lock(&kvm->arch.tdp_mmu_pages_lock);
+ 	list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots);
+@@ -1027,32 +1001,49 @@ void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm)
+ /*
+  * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that
+  * is about to be zapped, e.g. in response to a memslots update.  The actual
+- * zapping is performed asynchronously, so a reference is taken on all roots.
+- * Using a separate workqueue makes it easy to ensure that the destruction is
+- * performed before the "fast zap" completes, without keeping a separate list
+- * of invalidated roots; the list is effectively the list of work items in
+- * the workqueue.
+- *
+- * Get a reference even if the root is already invalid, the asynchronous worker
+- * assumes it was gifted a reference to the root it processes.  Because mmu_lock
+- * is held for write, it should be impossible to observe a root with zero refcount,
+- * i.e. the list of roots cannot be stale.
++ * zapping is performed asynchronously.  Using a separate workqueue makes it
++ * easy to ensure that the destruction is performed before the "fast zap"
++ * completes, without keeping a separate list of invalidated roots; the list is
++ * effectively the list of work items in the workqueue.
+  *
+- * This has essentially the same effect for the TDP MMU
+- * as updating mmu_valid_gen does for the shadow MMU.
++ * Note, the asynchronous worker is gifted the TDP MMU's reference.
++ * See kvm_tdp_mmu_get_vcpu_root_hpa().
+  */
+ void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm)
+ {
+ 	struct kvm_mmu_page *root;
+ 
+-	lockdep_assert_held_write(&kvm->mmu_lock);
+-	list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
+-		if (!root->role.invalid &&
+-		    !WARN_ON_ONCE(!kvm_tdp_mmu_get_root(root))) {
++	/*
++	 * mmu_lock must be held for write to ensure that a root doesn't become
++	 * invalid while there are active readers (invalidating a root while
++	 * there are active readers may or may not be problematic in practice,
++	 * but it's uncharted territory and not supported).
++	 *
++	 * Waive the assertion if there are no users of @kvm, i.e. the VM is
++	 * being destroyed after all references have been put, or if no vCPUs
++	 * have been created (which means there are no roots), i.e. the VM is
++	 * being destroyed in an error path of KVM_CREATE_VM.
++	 */
++	if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
++	    refcount_read(&kvm->users_count) && kvm->created_vcpus)
++		lockdep_assert_held_write(&kvm->mmu_lock);
++
++	/*
++	 * As above, mmu_lock isn't held when destroying the VM!  There can't
++	 * be other references to @kvm, i.e. nothing else can invalidate roots
++	 * or be consuming roots, but walking the list of roots does need to be
++	 * guarded against roots being deleted by the asynchronous zap worker.
++	 */
++	rcu_read_lock();
++
++	list_for_each_entry_rcu(root, &kvm->arch.tdp_mmu_roots, link) {
++		if (!root->role.invalid) {
+ 			root->role.invalid = true;
+ 			tdp_mmu_schedule_zap_root(kvm, root);
+ 		}
+ 	}
++
++	rcu_read_unlock();
+ }
+ 
+ /*
+diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
+index 4459cfbdbcb18..c2f0f74193f0e 100644
+--- a/drivers/block/ublk_drv.c
++++ b/drivers/block/ublk_drv.c
+@@ -1223,9 +1223,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
+ 			__func__, cmd->cmd_op, ub_cmd->q_id, tag,
+ 			ub_cmd->result);
+ 
+-	if (!(issue_flags & IO_URING_F_SQE128))
+-		goto out;
+-
+ 	if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues)
+ 		goto out;
+ 
+diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c
+index 4fb4fd4b06bda..737aa70e2cb3d 100644
+--- a/drivers/clk/clk-devres.c
++++ b/drivers/clk/clk-devres.c
+@@ -205,18 +205,19 @@ EXPORT_SYMBOL(devm_clk_put);
+ struct clk *devm_get_clk_from_child(struct device *dev,
+ 				    struct device_node *np, const char *con_id)
+ {
+-	struct clk **ptr, *clk;
++	struct devm_clk_state *state;
++	struct clk *clk;
+ 
+-	ptr = devres_alloc(devm_clk_release, sizeof(*ptr), GFP_KERNEL);
+-	if (!ptr)
++	state = devres_alloc(devm_clk_release, sizeof(*state), GFP_KERNEL);
++	if (!state)
+ 		return ERR_PTR(-ENOMEM);
+ 
+ 	clk = of_clk_get_by_name(np, con_id);
+ 	if (!IS_ERR(clk)) {
+-		*ptr = clk;
+-		devres_add(dev, ptr);
++		state->clk = clk;
++		devres_add(dev, state);
+ 	} else {
+-		devres_free(ptr);
++		devres_free(state);
+ 	}
+ 
+ 	return clk;
+diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c
+index 348b3a9170fa4..7f5ed1aa7a9f8 100644
+--- a/drivers/dma-buf/sw_sync.c
++++ b/drivers/dma-buf/sw_sync.c
+@@ -191,6 +191,7 @@ static const struct dma_fence_ops timeline_fence_ops = {
+  */
+ static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
+ {
++	LIST_HEAD(signalled);
+ 	struct sync_pt *pt, *next;
+ 
+ 	trace_sync_timeline(obj);
+@@ -203,21 +204,20 @@ static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
+ 		if (!timeline_fence_signaled(&pt->base))
+ 			break;
+ 
+-		list_del_init(&pt->link);
++		dma_fence_get(&pt->base);
++
++		list_move_tail(&pt->link, &signalled);
+ 		rb_erase(&pt->node, &obj->pt_tree);
+ 
+-		/*
+-		 * A signal callback may release the last reference to this
+-		 * fence, causing it to be freed. That operation has to be
+-		 * last to avoid a use after free inside this loop, and must
+-		 * be after we remove the fence from the timeline in order to
+-		 * prevent deadlocking on timeline->lock inside
+-		 * timeline_fence_release().
+-		 */
+ 		dma_fence_signal_locked(&pt->base);
+ 	}
+ 
+ 	spin_unlock_irq(&obj->lock);
++
++	list_for_each_entry_safe(pt, next, &signalled, link) {
++		list_del_init(&pt->link);
++		dma_fence_put(&pt->base);
++	}
+ }
+ 
+ /**
+diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c
+index fef12e57b1f13..b352775e5e0b8 100644
+--- a/drivers/gpio/gpio-sim.c
++++ b/drivers/gpio/gpio-sim.c
+@@ -290,6 +290,15 @@ static void gpio_sim_mutex_destroy(void *data)
+ 	mutex_destroy(lock);
+ }
+ 
++static void gpio_sim_dispose_mappings(void *data)
++{
++	struct gpio_sim_chip *chip = data;
++	unsigned int i;
++
++	for (i = 0; i < chip->gc.ngpio; i++)
++		irq_dispose_mapping(irq_find_mapping(chip->irq_sim, i));
++}
++
+ static void gpio_sim_sysfs_remove(void *data)
+ {
+ 	struct gpio_sim_chip *chip = data;
+@@ -398,10 +407,14 @@ static int gpio_sim_add_bank(struct fwnode_handle *swnode, struct device *dev)
+ 	if (!chip->pull_map)
+ 		return -ENOMEM;
+ 
+-	chip->irq_sim = devm_irq_domain_create_sim(dev, NULL, num_lines);
++	chip->irq_sim = devm_irq_domain_create_sim(dev, swnode, num_lines);
+ 	if (IS_ERR(chip->irq_sim))
+ 		return PTR_ERR(chip->irq_sim);
+ 
++	ret = devm_add_action_or_reset(dev, gpio_sim_dispose_mappings, chip);
++	if (ret)
++		return ret;
++
+ 	mutex_init(&chip->lock);
+ 	ret = devm_add_action_or_reset(dev, gpio_sim_mutex_destroy,
+ 				       &chip->lock);
+diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
+index a032003c340cc..d6ea47873627f 100644
+--- a/drivers/gpu/drm/arm/hdlcd_drv.c
++++ b/drivers/gpu/drm/arm/hdlcd_drv.c
+@@ -290,7 +290,7 @@ static int hdlcd_drm_bind(struct device *dev)
+ 	 */
+ 	if (hdlcd_read(hdlcd, HDLCD_REG_COMMAND)) {
+ 		hdlcd_write(hdlcd, HDLCD_REG_COMMAND, 0);
+-		drm_aperture_remove_framebuffers(false, &hdlcd_driver);
++		drm_aperture_remove_framebuffers(&hdlcd_driver);
+ 	}
+ 
+ 	drm_mode_config_reset(drm);
+diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
+index 142668cd6d7cd..688ba358f5319 100644
+--- a/drivers/gpu/drm/armada/armada_drv.c
++++ b/drivers/gpu/drm/armada/armada_drv.c
+@@ -95,7 +95,7 @@ static int armada_drm_bind(struct device *dev)
+ 	}
+ 
+ 	/* Remove early framebuffers */
+-	ret = drm_aperture_remove_framebuffers(false, &armada_drm_driver);
++	ret = drm_aperture_remove_framebuffers(&armada_drm_driver);
+ 	if (ret) {
+ 		dev_err(dev, "[" DRM_NAME ":%s] can't kick out simple-fb: %d\n",
+ 			__func__, ret);
+diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
+index b9392f31e6291..800471f2a2037 100644
+--- a/drivers/gpu/drm/ast/ast_drv.c
++++ b/drivers/gpu/drm/ast/ast_drv.c
+@@ -89,27 +89,13 @@ static const struct pci_device_id ast_pciidlist[] = {
+ 
+ MODULE_DEVICE_TABLE(pci, ast_pciidlist);
+ 
+-static int ast_remove_conflicting_framebuffers(struct pci_dev *pdev)
+-{
+-	bool primary = false;
+-	resource_size_t base, size;
+-
+-	base = pci_resource_start(pdev, 0);
+-	size = pci_resource_len(pdev, 0);
+-#ifdef CONFIG_X86
+-	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+-#endif
+-
+-	return drm_aperture_remove_conflicting_framebuffers(base, size, primary, &ast_driver);
+-}
+-
+ static int ast_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ {
+ 	struct ast_private *ast;
+ 	struct drm_device *dev;
+ 	int ret;
+ 
+-	ret = ast_remove_conflicting_framebuffers(pdev);
++	ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &ast_driver);
+ 	if (ret)
+ 		return ret;
+ 
+diff --git a/drivers/gpu/drm/drm_aperture.c b/drivers/gpu/drm/drm_aperture.c
+index 3b8fdeeafd53a..697cffbfd6037 100644
+--- a/drivers/gpu/drm/drm_aperture.c
++++ b/drivers/gpu/drm/drm_aperture.c
+@@ -32,17 +32,13 @@
+  *
+  *	static int remove_conflicting_framebuffers(struct pci_dev *pdev)
+  *	{
+- *		bool primary = false;
+  *		resource_size_t base, size;
+  *		int ret;
+  *
+  *		base = pci_resource_start(pdev, 0);
+  *		size = pci_resource_len(pdev, 0);
+- *	#ifdef CONFIG_X86
+- *		primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+- *	#endif
+  *
+- *		return drm_aperture_remove_conflicting_framebuffers(base, size, primary,
++ *		return drm_aperture_remove_conflicting_framebuffers(base, size,
+  *		                                                    &example_driver);
+  *	}
+  *
+@@ -161,7 +157,6 @@ EXPORT_SYMBOL(devm_aperture_acquire_from_firmware);
+  * drm_aperture_remove_conflicting_framebuffers - remove existing framebuffers in the given range
+  * @base: the aperture's base address in physical memory
+  * @size: aperture size in bytes
+- * @primary: also kick vga16fb if present
+  * @req_driver: requesting DRM driver
+  *
+  * This function removes graphics device drivers which use the memory range described by
+@@ -171,9 +166,9 @@ EXPORT_SYMBOL(devm_aperture_acquire_from_firmware);
+  * 0 on success, or a negative errno code otherwise
+  */
+ int drm_aperture_remove_conflicting_framebuffers(resource_size_t base, resource_size_t size,
+-						 bool primary, const struct drm_driver *req_driver)
++						 const struct drm_driver *req_driver)
+ {
+-	return aperture_remove_conflicting_devices(base, size, primary, req_driver->name);
++	return aperture_remove_conflicting_devices(base, size, false, req_driver->name);
+ }
+ EXPORT_SYMBOL(drm_aperture_remove_conflicting_framebuffers);
+ 
+diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
+index cd9c73f5a64ab..738eb558a97e9 100644
+--- a/drivers/gpu/drm/gma500/psb_drv.c
++++ b/drivers/gpu/drm/gma500/psb_drv.c
+@@ -424,12 +424,17 @@ static int psb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+ 
+ 	/*
+ 	 * We cannot yet easily find the framebuffer's location in memory. So
+-	 * remove all framebuffers here.
++	 * remove all framebuffers here. Note that we still want the pci special
++	 * handling to kick out vgacon.
+ 	 *
+ 	 * TODO: Refactor psb_driver_load() to map vdc_reg earlier. Then we
+ 	 *       might be able to read the framebuffer range from the device.
+ 	 */
+-	ret = drm_aperture_remove_framebuffers(true, &driver);
++	ret = drm_aperture_remove_framebuffers(&driver);
++	if (ret)
++		return ret;
++
++	ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &driver);
+ 	if (ret)
+ 		return ret;
+ 
+diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
+index ca127ff797f75..29ee0814bccc8 100644
+--- a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
++++ b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
+@@ -74,7 +74,6 @@ static int hyperv_setup_vram(struct hyperv_drm_device *hv,
+ 
+ 	drm_aperture_remove_conflicting_framebuffers(screen_info.lfb_base,
+ 						     screen_info.lfb_size,
+-						     false,
+ 						     &hyperv_driver);
+ 
+ 	hv->fb_size = (unsigned long)hv->mmio_megabytes * 1024 * 1024;
+diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
+index b2838732ac936..cc84685368715 100644
+--- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
++++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
+@@ -165,14 +165,60 @@ static u32 preparser_disable(bool state)
+ 	return MI_ARB_CHECK | 1 << 8 | state;
+ }
+ 
+-u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg)
++static i915_reg_t gen12_get_aux_inv_reg(struct intel_engine_cs *engine)
+ {
+-	u32 gsi_offset = gt->uncore->gsi_offset;
++	switch (engine->id) {
++	case RCS0:
++		return GEN12_CCS_AUX_INV;
++	case BCS0:
++		return GEN12_BCS0_AUX_INV;
++	case VCS0:
++		return GEN12_VD0_AUX_INV;
++	case VCS2:
++		return GEN12_VD2_AUX_INV;
++	case VECS0:
++		return GEN12_VE0_AUX_INV;
++	case CCS0:
++		return GEN12_CCS0_AUX_INV;
++	default:
++		return INVALID_MMIO_REG;
++	}
++}
++
++static bool gen12_needs_ccs_aux_inv(struct intel_engine_cs *engine)
++{
++	i915_reg_t reg = gen12_get_aux_inv_reg(engine);
++
++	if (IS_PONTEVECCHIO(engine->i915))
++		return false;
++
++	/*
++	 * So far platforms supported by i915 having flat ccs do not require
++	 * AUX invalidation. Check also whether the engine requires it.
++	 */
++	return i915_mmio_reg_valid(reg) && !HAS_FLAT_CCS(engine->i915);
++}
++
++u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs)
++{
++	i915_reg_t inv_reg = gen12_get_aux_inv_reg(engine);
++	u32 gsi_offset = engine->gt->uncore->gsi_offset;
++
++	if (!gen12_needs_ccs_aux_inv(engine))
++		return cs;
+ 
+ 	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;
+ 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;
+ 	*cs++ = AUX_INV;
+-	*cs++ = MI_NOOP;
++
++	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |
++		MI_SEMAPHORE_REGISTER_POLL |
++		MI_SEMAPHORE_POLL |
++		MI_SEMAPHORE_SAD_EQ_SDD;
++	*cs++ = 0;
++	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;
++	*cs++ = 0;
++	*cs++ = 0;
+ 
+ 	return cs;
+ }
+@@ -181,7 +227,11 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
+ {
+ 	struct intel_engine_cs *engine = rq->engine;
+ 
+-	if (mode & EMIT_FLUSH) {
++	/*
++	 * On Aux CCS platforms the invalidation of the Aux
++	 * table requires quiescing memory traffic beforehand
++	 */
++	if (mode & EMIT_FLUSH || gen12_needs_ccs_aux_inv(engine)) {
+ 		u32 flags = 0;
+ 		u32 *cs;
+ 
+@@ -236,10 +286,9 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
+ 		else if (engine->class == COMPUTE_CLASS)
+ 			flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
+ 
+-		if (!HAS_FLAT_CCS(rq->engine->i915))
+-			count = 8 + 4;
+-		else
+-			count = 8;
++		count = 8;
++		if (gen12_needs_ccs_aux_inv(rq->engine))
++			count += 8;
+ 
+ 		cs = intel_ring_begin(rq, count);
+ 		if (IS_ERR(cs))
+@@ -254,11 +303,7 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
+ 
+ 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);
+ 
+-		if (!HAS_FLAT_CCS(rq->engine->i915)) {
+-			/* hsdes: 1809175790 */
+-			cs = gen12_emit_aux_table_inv(rq->engine->gt, cs,
+-						      GEN12_CCS_AUX_INV);
+-		}
++		cs = gen12_emit_aux_table_inv(engine, cs);
+ 
+ 		*cs++ = preparser_disable(false);
+ 		intel_ring_advance(rq, cs);
+@@ -269,21 +314,14 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
+ 
+ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode)
+ {
+-	intel_engine_mask_t aux_inv = 0;
+-	u32 cmd, *cs;
++	u32 cmd = 4;
++	u32 *cs;
+ 
+-	cmd = 4;
+ 	if (mode & EMIT_INVALIDATE) {
+ 		cmd += 2;
+ 
+-		if (!HAS_FLAT_CCS(rq->engine->i915) &&
+-		    (rq->engine->class == VIDEO_DECODE_CLASS ||
+-		     rq->engine->class == VIDEO_ENHANCEMENT_CLASS)) {
+-			aux_inv = rq->engine->mask &
+-				~GENMASK(_BCS(I915_MAX_BCS - 1), BCS0);
+-			if (aux_inv)
+-				cmd += 4;
+-		}
++		if (gen12_needs_ccs_aux_inv(rq->engine))
++			cmd += 8;
+ 	}
+ 
+ 	cs = intel_ring_begin(rq, cmd);
+@@ -314,14 +352,7 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode)
+ 	*cs++ = 0; /* upper addr */
+ 	*cs++ = 0; /* value */
+ 
+-	if (aux_inv) { /* hsdes: 1809175790 */
+-		if (rq->engine->class == VIDEO_DECODE_CLASS)
+-			cs = gen12_emit_aux_table_inv(rq->engine->gt,
+-						      cs, GEN12_VD0_AUX_INV);
+-		else
+-			cs = gen12_emit_aux_table_inv(rq->engine->gt,
+-						      cs, GEN12_VE0_AUX_INV);
+-	}
++	cs = gen12_emit_aux_table_inv(rq->engine, cs);
+ 
+ 	if (mode & EMIT_INVALIDATE)
+ 		*cs++ = preparser_disable(false);
+diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.h b/drivers/gpu/drm/i915/gt/gen8_engine_cs.h
+index e4d24c811dd61..651eb786e930c 100644
+--- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.h
++++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.h
+@@ -13,6 +13,7 @@
+ #include "intel_gt_regs.h"
+ #include "intel_gpu_commands.h"
+ 
++struct intel_engine_cs;
+ struct intel_gt;
+ struct i915_request;
+ 
+@@ -46,7 +47,7 @@ u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);
+ u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);
+ u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);
+ 
+-u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg);
++u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs);
+ 
+ static inline u32 *
+ __gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset)
+diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h
+index d4e9702d3c8e7..25ea5f8a464a4 100644
+--- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h
++++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h
+@@ -120,6 +120,7 @@
+ #define   MI_SEMAPHORE_TARGET(engine)	((engine)<<15)
+ #define MI_SEMAPHORE_WAIT	MI_INSTR(0x1c, 2) /* GEN8+ */
+ #define MI_SEMAPHORE_WAIT_TOKEN	MI_INSTR(0x1c, 3) /* GEN12+ */
++#define   MI_SEMAPHORE_REGISTER_POLL	(1 << 16)
+ #define   MI_SEMAPHORE_POLL		(1 << 15)
+ #define   MI_SEMAPHORE_SAD_GT_SDD	(0 << 12)
+ #define   MI_SEMAPHORE_SAD_GTE_SDD	(1 << 12)
+diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
+index 137e41e37ea54..7eb01ff17d89b 100644
+--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
++++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
+@@ -1296,10 +1296,7 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs)
+ 	    IS_DG2_G11(ce->engine->i915))
+ 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE, 0);
+ 
+-	/* hsdes: 1809175790 */
+-	if (!HAS_FLAT_CCS(ce->engine->i915))
+-		cs = gen12_emit_aux_table_inv(ce->engine->gt,
+-					      cs, GEN12_CCS_AUX_INV);
++	cs = gen12_emit_aux_table_inv(ce->engine, cs);
+ 
+ 	/* Wa_16014892111 */
+ 	if (IS_DG2(ce->engine->i915))
+@@ -1322,17 +1319,7 @@ gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs)
+ 						    PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE,
+ 						    0);
+ 
+-	/* hsdes: 1809175790 */
+-	if (!HAS_FLAT_CCS(ce->engine->i915)) {
+-		if (ce->engine->class == VIDEO_DECODE_CLASS)
+-			cs = gen12_emit_aux_table_inv(ce->engine->gt,
+-						      cs, GEN12_VD0_AUX_INV);
+-		else if (ce->engine->class == VIDEO_ENHANCEMENT_CLASS)
+-			cs = gen12_emit_aux_table_inv(ce->engine->gt,
+-						      cs, GEN12_VE0_AUX_INV);
+-	}
+-
+-	return cs;
++	return gen12_emit_aux_table_inv(ce->engine, cs);
+ }
+ 
+ static void
+diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
+index 35bc2a3fa811c..75a93951fe429 100644
+--- a/drivers/gpu/drm/i915/i915_driver.c
++++ b/drivers/gpu/drm/i915/i915_driver.c
+@@ -574,7 +574,6 @@ static int i915_pcode_init(struct drm_i915_private *i915)
+ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
+ {
+ 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
+-	struct pci_dev *root_pdev;
+ 	int ret;
+ 
+ 	if (i915_inject_probe_failure(dev_priv))
+@@ -686,15 +685,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
+ 
+ 	intel_bw_init_hw(dev_priv);
+ 
+-	/*
+-	 * FIXME: Temporary hammer to avoid freezing the machine on our DGFX
+-	 * This should be totally removed when we handle the pci states properly
+-	 * on runtime PM and on s2idle cases.
+-	 */
+-	root_pdev = pcie_find_root_port(pdev);
+-	if (root_pdev)
+-		pci_d3cold_disable(root_pdev);
+-
+ 	return 0;
+ 
+ err_msi:
+@@ -718,16 +708,11 @@ err_perf:
+ static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
+ {
+ 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
+-	struct pci_dev *root_pdev;
+ 
+ 	i915_perf_fini(dev_priv);
+ 
+ 	if (pdev->msi_enabled)
+ 		pci_disable_msi(pdev);
+-
+-	root_pdev = pcie_find_root_port(pdev);
+-	if (root_pdev)
+-		pci_d3cold_enable(root_pdev);
+ }
+ 
+ /**
+@@ -1625,6 +1610,8 @@ static int intel_runtime_suspend(struct device *kdev)
+ {
+ 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
+ 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
++	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
++	struct pci_dev *root_pdev;
+ 	struct intel_gt *gt;
+ 	int ret, i;
+ 
+@@ -1674,6 +1661,15 @@ static int intel_runtime_suspend(struct device *kdev)
+ 		drm_err(&dev_priv->drm,
+ 			"Unclaimed access detected prior to suspending\n");
+ 
++	/*
++	 * FIXME: Temporary hammer to avoid freezing the machine on our DGFX
++	 * This should be totally removed when we handle the pci states properly
++	 * on runtime PM.
++	 */
++	root_pdev = pcie_find_root_port(pdev);
++	if (root_pdev)
++		pci_d3cold_disable(root_pdev);
++
+ 	rpm->suspended = true;
+ 
+ 	/*
+@@ -1712,6 +1708,8 @@ static int intel_runtime_resume(struct device *kdev)
+ {
+ 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
+ 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
++	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
++	struct pci_dev *root_pdev;
+ 	struct intel_gt *gt;
+ 	int ret, i;
+ 
+@@ -1725,6 +1723,11 @@ static int intel_runtime_resume(struct device *kdev)
+ 
+ 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
+ 	rpm->suspended = false;
++
++	root_pdev = pcie_find_root_port(pdev);
++	if (root_pdev)
++		pci_d3cold_enable(root_pdev);
++
+ 	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))
+ 		drm_dbg(&dev_priv->drm,
+ 			"Unclaimed access during suspend, bios?\n");
+diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
+index eea433ade79d0..119544d88b586 100644
+--- a/drivers/gpu/drm/meson/meson_drv.c
++++ b/drivers/gpu/drm/meson/meson_drv.c
+@@ -285,7 +285,7 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
+ 	 * Remove early framebuffers (ie. simplefb). The framebuffer can be
+ 	 * located anywhere in RAM
+ 	 */
+-	ret = drm_aperture_remove_framebuffers(false, &meson_driver);
++	ret = drm_aperture_remove_framebuffers(&meson_driver);
+ 	if (ret)
+ 		goto free_drm;
+ 
+diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
+index 46168eccfac4a..d4a9b501e1bcc 100644
+--- a/drivers/gpu/drm/msm/msm_fbdev.c
++++ b/drivers/gpu/drm/msm/msm_fbdev.c
+@@ -157,7 +157,7 @@ struct drm_fb_helper *msm_fbdev_init(struct drm_device *dev)
+ 	}
+ 
+ 	/* the fw fb could be anywhere in memory */
+-	ret = drm_aperture_remove_framebuffers(false, dev->driver);
++	ret = drm_aperture_remove_framebuffers(dev->driver);
+ 	if (ret)
+ 		goto fini;
+ 
+diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+index 813f9f8c86982..8e12053a220b0 100644
+--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
++++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+@@ -140,7 +140,7 @@ static int rockchip_drm_bind(struct device *dev)
+ 	int ret;
+ 
+ 	/* Remove existing drivers that may own the framebuffer memory. */
+-	ret = drm_aperture_remove_framebuffers(false, &rockchip_drm_driver);
++	ret = drm_aperture_remove_framebuffers(&rockchip_drm_driver);
+ 	if (ret) {
+ 		DRM_DEV_ERROR(dev,
+ 			      "Failed to remove existing framebuffers - %d.\n",
+diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
+index d7914f5122dff..0a09a85ac9d69 100644
+--- a/drivers/gpu/drm/stm/drv.c
++++ b/drivers/gpu/drm/stm/drv.c
+@@ -185,7 +185,7 @@ static int stm_drm_platform_probe(struct platform_device *pdev)
+ 
+ 	DRM_DEBUG("%s\n", __func__);
+ 
+-	ret = drm_aperture_remove_framebuffers(false, &drv_driver);
++	ret = drm_aperture_remove_framebuffers(&drv_driver);
+ 	if (ret)
+ 		return ret;
+ 
+diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
+index 7910c5853f0a8..5c483bbccbbbc 100644
+--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
++++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
+@@ -98,7 +98,7 @@ static int sun4i_drv_bind(struct device *dev)
+ 		goto unbind_all;
+ 
+ 	/* Remove early framebuffers (ie. simplefb) */
+-	ret = drm_aperture_remove_framebuffers(false, &sun4i_drv_driver);
++	ret = drm_aperture_remove_framebuffers(&sun4i_drv_driver);
+ 	if (ret)
+ 		goto unbind_all;
+ 
+diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
+index a1f909dac89a7..5fc55b9777cbf 100644
+--- a/drivers/gpu/drm/tegra/drm.c
++++ b/drivers/gpu/drm/tegra/drm.c
+@@ -1252,7 +1252,7 @@ static int host1x_drm_probe(struct host1x_device *dev)
+ 
+ 	drm_mode_config_reset(drm);
+ 
+-	err = drm_aperture_remove_framebuffers(false, &tegra_drm_driver);
++	err = drm_aperture_remove_framebuffers(&tegra_drm_driver);
+ 	if (err < 0)
+ 		goto hub;
+ 
+diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
+index 8c329c071c62d..b6384a5dfdbc1 100644
+--- a/drivers/gpu/drm/vc4/vc4_drv.c
++++ b/drivers/gpu/drm/vc4/vc4_drv.c
+@@ -351,7 +351,7 @@ static int vc4_drm_bind(struct device *dev)
+ 			return -EPROBE_DEFER;
+ 	}
+ 
+-	ret = drm_aperture_remove_framebuffers(false, driver);
++	ret = drm_aperture_remove_framebuffers(driver);
+ 	if (ret)
+ 		return ret;
+ 
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+index 1ec9c53a7bf43..8459fab9d9797 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+@@ -1683,4 +1683,16 @@ static inline bool vmw_has_fences(struct vmw_private *vmw)
+ 	return (vmw_fifo_caps(vmw) & SVGA_FIFO_CAP_FENCE) != 0;
+ }
+ 
++static inline bool vmw_shadertype_is_valid(enum vmw_sm_type shader_model,
++					   u32 shader_type)
++{
++	SVGA3dShaderType max_allowed = SVGA3D_SHADERTYPE_PREDX_MAX;
++
++	if (shader_model >= VMW_SM_5)
++		max_allowed = SVGA3D_SHADERTYPE_MAX;
++	else if (shader_model >= VMW_SM_4)
++		max_allowed = SVGA3D_SHADERTYPE_DX10_MAX;
++	return shader_type >= SVGA3D_SHADERTYPE_MIN && shader_type < max_allowed;
++}
++
+ #endif
+diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+index 1c88b74d68cf0..58ca9adf09871 100644
+--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
++++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+@@ -1985,7 +1985,7 @@ static int vmw_cmd_set_shader(struct vmw_private *dev_priv,
+ 
+ 	cmd = container_of(header, typeof(*cmd), header);
+ 
+-	if (cmd->body.type >= SVGA3D_SHADERTYPE_PREDX_MAX) {
++	if (!vmw_shadertype_is_valid(VMW_SM_LEGACY, cmd->body.type)) {
+ 		VMW_DEBUG_USER("Illegal shader type %u.\n",
+ 			       (unsigned int) cmd->body.type);
+ 		return -EINVAL;
+@@ -2108,8 +2108,6 @@ vmw_cmd_dx_set_single_constant_buffer(struct vmw_private *dev_priv,
+ 				      SVGA3dCmdHeader *header)
+ {
+ 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetSingleConstantBuffer);
+-	SVGA3dShaderType max_shader_num = has_sm5_context(dev_priv) ?
+-		SVGA3D_NUM_SHADERTYPE : SVGA3D_NUM_SHADERTYPE_DX10;
+ 
+ 	struct vmw_resource *res = NULL;
+ 	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
+@@ -2126,6 +2124,14 @@ vmw_cmd_dx_set_single_constant_buffer(struct vmw_private *dev_priv,
+ 	if (unlikely(ret != 0))
+ 		return ret;
+ 
++	if (!vmw_shadertype_is_valid(dev_priv->sm_type, cmd->body.type) ||
++	    cmd->body.slot >= SVGA3D_DX_MAX_CONSTBUFFERS) {
++		VMW_DEBUG_USER("Illegal const buffer shader %u slot %u.\n",
++			       (unsigned int) cmd->body.type,
++			       (unsigned int) cmd->body.slot);
++		return -EINVAL;
++	}
++
+ 	binding.bi.ctx = ctx_node->ctx;
+ 	binding.bi.res = res;
+ 	binding.bi.bt = vmw_ctx_binding_cb;
+@@ -2134,14 +2140,6 @@ vmw_cmd_dx_set_single_constant_buffer(struct vmw_private *dev_priv,
+ 	binding.size = cmd->body.sizeInBytes;
+ 	binding.slot = cmd->body.slot;
+ 
+-	if (binding.shader_slot >= max_shader_num ||
+-	    binding.slot >= SVGA3D_DX_MAX_CONSTBUFFERS) {
+-		VMW_DEBUG_USER("Illegal const buffer shader %u slot %u.\n",
+-			       (unsigned int) cmd->body.type,
+-			       (unsigned int) binding.slot);
+-		return -EINVAL;
+-	}
+-
+ 	vmw_binding_add(ctx_node->staged, &binding.bi, binding.shader_slot,
+ 			binding.slot);
+ 
+@@ -2200,15 +2198,13 @@ static int vmw_cmd_dx_set_shader_res(struct vmw_private *dev_priv,
+ {
+ 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetShaderResources) =
+ 		container_of(header, typeof(*cmd), header);
+-	SVGA3dShaderType max_allowed = has_sm5_context(dev_priv) ?
+-		SVGA3D_SHADERTYPE_MAX : SVGA3D_SHADERTYPE_DX10_MAX;
+ 
+ 	u32 num_sr_view = (cmd->header.size - sizeof(cmd->body)) /
+ 		sizeof(SVGA3dShaderResourceViewId);
+ 
+ 	if ((u64) cmd->body.startView + (u64) num_sr_view >
+ 	    (u64) SVGA3D_DX_MAX_SRVIEWS ||
+-	    cmd->body.type >= max_allowed) {
++	    !vmw_shadertype_is_valid(dev_priv->sm_type, cmd->body.type)) {
+ 		VMW_DEBUG_USER("Invalid shader binding.\n");
+ 		return -EINVAL;
+ 	}
+@@ -2232,8 +2228,6 @@ static int vmw_cmd_dx_set_shader(struct vmw_private *dev_priv,
+ 				 SVGA3dCmdHeader *header)
+ {
+ 	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetShader);
+-	SVGA3dShaderType max_allowed = has_sm5_context(dev_priv) ?
+-		SVGA3D_SHADERTYPE_MAX : SVGA3D_SHADERTYPE_DX10_MAX;
+ 	struct vmw_resource *res = NULL;
+ 	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
+ 	struct vmw_ctx_bindinfo_shader binding;
+@@ -2244,8 +2238,7 @@ static int vmw_cmd_dx_set_shader(struct vmw_private *dev_priv,
+ 
+ 	cmd = container_of(header, typeof(*cmd), header);
+ 
+-	if (cmd->body.type >= max_allowed ||
+-	    cmd->body.type < SVGA3D_SHADERTYPE_MIN) {
++	if (!vmw_shadertype_is_valid(dev_priv->sm_type, cmd->body.type)) {
+ 		VMW_DEBUG_USER("Illegal shader type %u.\n",
+ 			       (unsigned int) cmd->body.type);
+ 		return -EINVAL;
+diff --git a/drivers/hwmon/aquacomputer_d5next.c b/drivers/hwmon/aquacomputer_d5next.c
+index c51a2678f0eb5..8c7796d3fdd2d 100644
+--- a/drivers/hwmon/aquacomputer_d5next.c
++++ b/drivers/hwmon/aquacomputer_d5next.c
+@@ -12,9 +12,11 @@
+ 
+ #include <linux/crc16.h>
+ #include <linux/debugfs.h>
++#include <linux/delay.h>
+ #include <linux/hid.h>
+ #include <linux/hwmon.h>
+ #include <linux/jiffies.h>
++#include <linux/ktime.h>
+ #include <linux/module.h>
+ #include <linux/mutex.h>
+ #include <linux/seq_file.h>
+@@ -49,6 +51,8 @@ static const char *const aqc_device_names[] = {
+ 
+ #define CTRL_REPORT_ID			0x03
+ 
++#define CTRL_REPORT_DELAY		200	/* ms */
++
+ /* The HID report that the official software always sends
+  * after writing values, currently same for all devices
+  */
+@@ -269,6 +273,9 @@ struct aqc_data {
+ 	enum kinds kind;
+ 	const char *name;
+ 
++	ktime_t last_ctrl_report_op;
++	int ctrl_report_delay;	/* Delay between two ctrl report operations, in ms */
++
+ 	int buffer_size;
+ 	u8 *buffer;
+ 	int checksum_start;
+@@ -325,17 +332,35 @@ static int aqc_pwm_to_percent(long val)
+ 	return DIV_ROUND_CLOSEST(val * 100 * 100, 255);
+ }
+ 
++static void aqc_delay_ctrl_report(struct aqc_data *priv)
++{
++	/*
++	 * If previous read or write is too close to this one, delay the current operation
++	 * to give the device enough time to process the previous one.
++	 */
++	if (priv->ctrl_report_delay) {
++		s64 delta = ktime_ms_delta(ktime_get(), priv->last_ctrl_report_op);
++
++		if (delta < priv->ctrl_report_delay)
++			msleep(priv->ctrl_report_delay - delta);
++	}
++}
++
+ /* Expects the mutex to be locked */
+ static int aqc_get_ctrl_data(struct aqc_data *priv)
+ {
+ 	int ret;
+ 
++	aqc_delay_ctrl_report(priv);
++
+ 	memset(priv->buffer, 0x00, priv->buffer_size);
+ 	ret = hid_hw_raw_request(priv->hdev, CTRL_REPORT_ID, priv->buffer, priv->buffer_size,
+ 				 HID_FEATURE_REPORT, HID_REQ_GET_REPORT);
+ 	if (ret < 0)
+ 		ret = -ENODATA;
+ 
++	priv->last_ctrl_report_op = ktime_get();
++
+ 	return ret;
+ }
+ 
+@@ -345,6 +370,8 @@ static int aqc_send_ctrl_data(struct aqc_data *priv)
+ 	int ret;
+ 	u16 checksum;
+ 
++	aqc_delay_ctrl_report(priv);
++
+ 	/* Init and xorout value for CRC-16/USB is 0xffff */
+ 	checksum = crc16(0xffff, priv->buffer + priv->checksum_start, priv->checksum_length);
+ 	checksum ^= 0xffff;
+@@ -356,12 +383,16 @@ static int aqc_send_ctrl_data(struct aqc_data *priv)
+ 	ret = hid_hw_raw_request(priv->hdev, CTRL_REPORT_ID, priv->buffer, priv->buffer_size,
+ 				 HID_FEATURE_REPORT, HID_REQ_SET_REPORT);
+ 	if (ret < 0)
+-		return ret;
++		goto record_access_and_ret;
+ 
+ 	/* The official software sends this report after every change, so do it here as well */
+ 	ret = hid_hw_raw_request(priv->hdev, SECONDARY_CTRL_REPORT_ID, secondary_ctrl_report,
+ 				 SECONDARY_CTRL_REPORT_SIZE, HID_FEATURE_REPORT,
+ 				 HID_REQ_SET_REPORT);
++
++record_access_and_ret:
++	priv->last_ctrl_report_op = ktime_get();
++
+ 	return ret;
+ }
+ 
+@@ -853,6 +884,7 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ 		priv->virtual_temp_sensor_start_offset = D5NEXT_VIRTUAL_SENSORS_START;
+ 		priv->power_cycle_count_offset = D5NEXT_POWER_CYCLES;
+ 		priv->buffer_size = D5NEXT_CTRL_REPORT_SIZE;
++		priv->ctrl_report_delay = CTRL_REPORT_DELAY;
+ 
+ 		priv->temp_label = label_d5next_temp;
+ 		priv->virtual_temp_label = label_virtual_temp_sensors;
+@@ -893,6 +925,7 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ 		priv->virtual_temp_sensor_start_offset = OCTO_VIRTUAL_SENSORS_START;
+ 		priv->power_cycle_count_offset = OCTO_POWER_CYCLES;
+ 		priv->buffer_size = OCTO_CTRL_REPORT_SIZE;
++		priv->ctrl_report_delay = CTRL_REPORT_DELAY;
+ 
+ 		priv->temp_label = label_temp_sensors;
+ 		priv->virtual_temp_label = label_virtual_temp_sensors;
+@@ -913,6 +946,7 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
+ 		priv->virtual_temp_sensor_start_offset = QUADRO_VIRTUAL_SENSORS_START;
+ 		priv->power_cycle_count_offset = QUADRO_POWER_CYCLES;
+ 		priv->buffer_size = QUADRO_CTRL_REPORT_SIZE;
++		priv->ctrl_report_delay = CTRL_REPORT_DELAY;
+ 		priv->flow_sensor_offset = QUADRO_FLOW_SENSOR_OFFSET;
+ 
+ 		priv->temp_label = label_temp_sensors;
+diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c
+index d810a78dde51d..31e3c37662185 100644
+--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c
++++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c
+@@ -821,6 +821,8 @@ static int vb2ops_venc_queue_setup(struct vb2_queue *vq,
+ 		return -EINVAL;
+ 
+ 	if (*nplanes) {
++		if (*nplanes != q_data->fmt->num_planes)
++			return -EINVAL;
+ 		for (i = 0; i < *nplanes; i++)
+ 			if (sizes[i] < q_data->sizeimage[i])
+ 				return -EINVAL;
+diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c
+index b9dbad3a8af82..fc5da5d7744da 100644
+--- a/drivers/net/bonding/bond_alb.c
++++ b/drivers/net/bonding/bond_alb.c
+@@ -660,10 +660,10 @@ static struct slave *rlb_arp_xmit(struct sk_buff *skb, struct bonding *bond)
+ 		return NULL;
+ 	arp = (struct arp_pkt *)skb_network_header(skb);
+ 
+-	/* Don't modify or load balance ARPs that do not originate locally
+-	 * (e.g.,arrive via a bridge).
++	/* Don't modify or load balance ARPs that do not originate
++	 * from the bond itself or a VLAN directly above the bond.
+ 	 */
+-	if (!bond_slave_has_mac_rx(bond, arp->mac_src))
++	if (!bond_slave_has_mac_rcu(bond, arp->mac_src))
+ 		return NULL;
+ 
+ 	dev = ip_dev_find(dev_net(bond->dev), arp->ip_src);
+diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c
+index 26a472d2ea583..6d549dbdb4674 100644
+--- a/drivers/net/can/vxcan.c
++++ b/drivers/net/can/vxcan.c
+@@ -192,12 +192,7 @@ static int vxcan_newlink(struct net *net, struct net_device *dev,
+ 
+ 		nla_peer = data[VXCAN_INFO_PEER];
+ 		ifmp = nla_data(nla_peer);
+-		err = rtnl_nla_parse_ifla(peer_tb,
+-					  nla_data(nla_peer) +
+-					  sizeof(struct ifinfomsg),
+-					  nla_len(nla_peer) -
+-					  sizeof(struct ifinfomsg),
+-					  NULL);
++		err = rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack);
+ 		if (err < 0)
+ 			return err;
+ 
+diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
+index 51d2ef0dc835c..b988c8a40d536 100644
+--- a/drivers/net/dsa/mt7530.c
++++ b/drivers/net/dsa/mt7530.c
+@@ -1005,6 +1005,10 @@ mt753x_trap_frames(struct mt7530_priv *priv)
+ 	mt7530_rmw(priv, MT753X_BPC, MT753X_BPDU_PORT_FW_MASK,
+ 		   MT753X_BPDU_CPU_ONLY);
+ 
++	/* Trap 802.1X PAE frames to the CPU port(s) */
++	mt7530_rmw(priv, MT753X_BPC, MT753X_PAE_PORT_FW_MASK,
++		   MT753X_PAE_PORT_FW(MT753X_BPDU_CPU_ONLY));
++
+ 	/* Trap LLDP frames with :0E MAC DA to the CPU port(s) */
+ 	mt7530_rmw(priv, MT753X_RGAC2, MT753X_R0E_PORT_FW_MASK,
+ 		   MT753X_R0E_PORT_FW(MT753X_BPDU_CPU_ONLY));
+diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
+index 9a45663d8b4ef..6202b0f8c3f34 100644
+--- a/drivers/net/dsa/mt7530.h
++++ b/drivers/net/dsa/mt7530.h
+@@ -64,6 +64,8 @@ enum mt753x_id {
+ /* Registers for BPDU and PAE frame control*/
+ #define MT753X_BPC			0x24
+ #define  MT753X_BPDU_PORT_FW_MASK	GENMASK(2, 0)
++#define  MT753X_PAE_PORT_FW_MASK	GENMASK(18, 16)
++#define  MT753X_PAE_PORT_FW(x)		FIELD_PREP(MT753X_PAE_PORT_FW_MASK, x)
+ 
+ /* Register for :03 and :0E MAC DA frame control */
+ #define MT753X_RGAC2			0x2c
+diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c
+index 5f6af0870dfd6..0186482194d20 100644
+--- a/drivers/net/dsa/ocelot/felix_vsc9959.c
++++ b/drivers/net/dsa/ocelot/felix_vsc9959.c
+@@ -1071,6 +1071,9 @@ static u64 vsc9959_tas_remaining_gate_len_ps(u64 gate_len_ns)
+ 	if (gate_len_ns == U64_MAX)
+ 		return U64_MAX;
+ 
++	if (gate_len_ns < VSC9959_TAS_MIN_GATE_LEN_NS)
++		return 0;
++
+ 	return (gate_len_ns - VSC9959_TAS_MIN_GATE_LEN_NS) * PSEC_PER_NSEC;
+ }
+ 
+diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
+index 10c7c232cc4ec..52ee3751187a2 100644
+--- a/drivers/net/ethernet/broadcom/bgmac.c
++++ b/drivers/net/ethernet/broadcom/bgmac.c
+@@ -1448,7 +1448,7 @@ int bgmac_phy_connect_direct(struct bgmac *bgmac)
+ 	int err;
+ 
+ 	phy_dev = fixed_phy_register(PHY_POLL, &fphy_status, NULL);
+-	if (!phy_dev || IS_ERR(phy_dev)) {
++	if (IS_ERR(phy_dev)) {
+ 		dev_err(bgmac->dev, "Failed to register fixed PHY device\n");
+ 		return -ENODEV;
+ 	}
+diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c
+index 1fe8038587ac8..1779ee524dac7 100644
+--- a/drivers/net/ethernet/broadcom/genet/bcmmii.c
++++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c
+@@ -608,7 +608,7 @@ static int bcmgenet_mii_pd_init(struct bcmgenet_priv *priv)
+ 		};
+ 
+ 		phydev = fixed_phy_register(PHY_POLL, &fphy_status, NULL);
+-		if (!phydev || IS_ERR(phydev)) {
++		if (IS_ERR(phydev)) {
+ 			dev_err(kdev, "failed to register fixed PHY device\n");
+ 			return -ENODEV;
+ 		}
+diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
+index c2e7037c7ba1c..7750702900fa6 100644
+--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
++++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
+@@ -1466,7 +1466,7 @@ static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
+ 	tp->write_seq = snd_isn;
+ 	tp->snd_nxt = snd_isn;
+ 	tp->snd_una = snd_isn;
+-	inet_sk(sk)->inet_id = get_random_u16();
++	atomic_set(&inet_sk(sk)->inet_id, get_random_u16());
+ 	assign_rxopt(sk, opt);
+ 
+ 	if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
+diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c
+index 5b96cd94dcd24..0b4ec6e41eb41 100644
+--- a/drivers/net/ethernet/ibm/ibmveth.c
++++ b/drivers/net/ethernet/ibm/ibmveth.c
+@@ -203,7 +203,7 @@ static inline void ibmveth_flush_buffer(void *addr, unsigned long length)
+ 	unsigned long offset;
+ 
+ 	for (offset = 0; offset < length; offset += SMP_CACHE_BYTES)
+-		asm("dcbfl %0,%1" :: "b" (addr), "r" (offset));
++		asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset));
+ }
+ 
+ /* replenish the buffers for a pool.  note that we don't need to
+diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
+index 0e01b1927c1c6..08ccf0024ce1a 100644
+--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
++++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
+@@ -2615,7 +2615,7 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
+ 			retval = i40e_correct_mac_vlan_filters
+ 				(vsi, &tmp_add_list, &tmp_del_list,
+ 				 vlan_filters);
+-		else
++		else if (pf->vf)
+ 			retval = i40e_correct_vf_mac_vlan_filters
+ 				(vsi, &tmp_add_list, &tmp_del_list,
+ 				 vlan_filters, pf->vf[vsi->vf_id].trusted);
+@@ -2788,7 +2788,8 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
+ 	}
+ 
+ 	/* if the VF is not trusted do not do promisc */
+-	if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
++	if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
++	    !pf->vf[vsi->vf_id].trusted) {
+ 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
+ 		goto out;
+ 	}
+diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c
+index e864634d66bc6..818eca6aa4a41 100644
+--- a/drivers/net/ethernet/intel/ice/ice_base.c
++++ b/drivers/net/ethernet/intel/ice/ice_base.c
+@@ -396,7 +396,8 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring)
+ 	/* Receive Packet Data Buffer Size.
+ 	 * The Packet Data Buffer Size is defined in 128 byte units.
+ 	 */
+-	rlan_ctx.dbuf = ring->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
++	rlan_ctx.dbuf = DIV_ROUND_UP(ring->rx_buf_len,
++				     BIT_ULL(ICE_RLAN_CTX_DBUF_S));
+ 
+ 	/* use 32 byte descriptors */
+ 	rlan_ctx.dsize = 1;
+diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c
+index b8c31bf721ad1..b719e9a771e36 100644
+--- a/drivers/net/ethernet/intel/ice/ice_sriov.c
++++ b/drivers/net/ethernet/intel/ice/ice_sriov.c
+@@ -1240,7 +1240,7 @@ int ice_set_vf_spoofchk(struct net_device *netdev, int vf_id, bool ena)
+ 	if (!vf)
+ 		return -EINVAL;
+ 
+-	ret = ice_check_vf_ready_for_reset(vf);
++	ret = ice_check_vf_ready_for_cfg(vf);
+ 	if (ret)
+ 		goto out_put_vf;
+ 
+@@ -1355,7 +1355,7 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
+ 		goto out_put_vf;
+ 	}
+ 
+-	ret = ice_check_vf_ready_for_reset(vf);
++	ret = ice_check_vf_ready_for_cfg(vf);
+ 	if (ret)
+ 		goto out_put_vf;
+ 
+@@ -1409,7 +1409,7 @@ int ice_set_vf_trust(struct net_device *netdev, int vf_id, bool trusted)
+ 		return -EOPNOTSUPP;
+ 	}
+ 
+-	ret = ice_check_vf_ready_for_reset(vf);
++	ret = ice_check_vf_ready_for_cfg(vf);
+ 	if (ret)
+ 		goto out_put_vf;
+ 
+@@ -1722,7 +1722,7 @@ ice_set_vf_port_vlan(struct net_device *netdev, int vf_id, u16 vlan_id, u8 qos,
+ 	if (!vf)
+ 		return -EINVAL;
+ 
+-	ret = ice_check_vf_ready_for_reset(vf);
++	ret = ice_check_vf_ready_for_cfg(vf);
+ 	if (ret)
+ 		goto out_put_vf;
+ 
+diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+index 71047fc341392..9dbe6e9bb1f79 100644
+--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c
++++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+@@ -185,25 +185,6 @@ int ice_check_vf_ready_for_cfg(struct ice_vf *vf)
+ 	return 0;
+ }
+ 
+-/**
+- * ice_check_vf_ready_for_reset - check if VF is ready to be reset
+- * @vf: VF to check if it's ready to be reset
+- *
+- * The purpose of this function is to ensure that the VF is not in reset,
+- * disabled, and is both initialized and active, thus enabling us to safely
+- * initialize another reset.
+- */
+-int ice_check_vf_ready_for_reset(struct ice_vf *vf)
+-{
+-	int ret;
+-
+-	ret = ice_check_vf_ready_for_cfg(vf);
+-	if (!ret && !test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states))
+-		ret = -EAGAIN;
+-
+-	return ret;
+-}
+-
+ /**
+  * ice_trigger_vf_reset - Reset a VF on HW
+  * @vf: pointer to the VF structure
+@@ -588,11 +569,17 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags)
+ 		return 0;
+ 	}
+ 
++	if (flags & ICE_VF_RESET_LOCK)
++		mutex_lock(&vf->cfg_lock);
++	else
++		lockdep_assert_held(&vf->cfg_lock);
++
+ 	if (ice_is_vf_disabled(vf)) {
+ 		vsi = ice_get_vf_vsi(vf);
+ 		if (!vsi) {
+ 			dev_dbg(dev, "VF is already removed\n");
+-			return -EINVAL;
++			err = -EINVAL;
++			goto out_unlock;
+ 		}
+ 		ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, vf->vf_id);
+ 
+@@ -601,14 +588,9 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags)
+ 
+ 		dev_dbg(dev, "VF is already disabled, there is no need for resetting it, telling VM, all is fine %d\n",
+ 			vf->vf_id);
+-		return 0;
++		goto out_unlock;
+ 	}
+ 
+-	if (flags & ICE_VF_RESET_LOCK)
+-		mutex_lock(&vf->cfg_lock);
+-	else
+-		lockdep_assert_held(&vf->cfg_lock);
+-
+ 	/* Set VF disable bit state here, before triggering reset */
+ 	set_bit(ICE_VF_STATE_DIS, vf->vf_states);
+ 	ice_trigger_vf_reset(vf, flags & ICE_VF_RESET_VFLR, false);
+diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h
+index e5bed85724622..9f7fcd8e5714b 100644
+--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h
++++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h
+@@ -214,7 +214,6 @@ u16 ice_get_num_vfs(struct ice_pf *pf);
+ struct ice_vsi *ice_get_vf_vsi(struct ice_vf *vf);
+ bool ice_is_vf_disabled(struct ice_vf *vf);
+ int ice_check_vf_ready_for_cfg(struct ice_vf *vf);
+-int ice_check_vf_ready_for_reset(struct ice_vf *vf);
+ void ice_set_vf_state_dis(struct ice_vf *vf);
+ bool ice_is_any_vf_in_unicast_promisc(struct ice_pf *pf);
+ void
+diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c
+index ef3c709d6a750..2b4c791b6cbad 100644
+--- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c
++++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c
+@@ -3722,7 +3722,6 @@ error_handler:
+ 		ice_vc_notify_vf_link_state(vf);
+ 		break;
+ 	case VIRTCHNL_OP_RESET_VF:
+-		clear_bit(ICE_VF_STATE_ACTIVE, vf->vf_states);
+ 		ops->reset_vf(vf);
+ 		break;
+ 	case VIRTCHNL_OP_ADD_ETH_ADDR:
+diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c
+index 15e57460e19ea..07171e574e7d7 100644
+--- a/drivers/net/ethernet/intel/igb/igb_ptp.c
++++ b/drivers/net/ethernet/intel/igb/igb_ptp.c
+@@ -1404,18 +1404,6 @@ void igb_ptp_init(struct igb_adapter *adapter)
+ 		return;
+ 	}
+ 
+-	spin_lock_init(&adapter->tmreg_lock);
+-	INIT_WORK(&adapter->ptp_tx_work, igb_ptp_tx_work);
+-
+-	if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
+-		INIT_DELAYED_WORK(&adapter->ptp_overflow_work,
+-				  igb_ptp_overflow_check);
+-
+-	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
+-	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
+-
+-	igb_ptp_reset(adapter);
+-
+ 	adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
+ 						&adapter->pdev->dev);
+ 	if (IS_ERR(adapter->ptp_clock)) {
+@@ -1425,6 +1413,18 @@ void igb_ptp_init(struct igb_adapter *adapter)
+ 		dev_info(&adapter->pdev->dev, "added PHC on %s\n",
+ 			 adapter->netdev->name);
+ 		adapter->ptp_flags |= IGB_PTP_ENABLED;
++
++		spin_lock_init(&adapter->tmreg_lock);
++		INIT_WORK(&adapter->ptp_tx_work, igb_ptp_tx_work);
++
++		if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
++			INIT_DELAYED_WORK(&adapter->ptp_overflow_work,
++					  igb_ptp_overflow_check);
++
++		adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
++		adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
++
++		igb_ptp_reset(adapter);
+ 	}
+ }
+ 
+diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h
+index dbfa4b9dee066..90ca01889cd82 100644
+--- a/drivers/net/ethernet/intel/igc/igc_defines.h
++++ b/drivers/net/ethernet/intel/igc/igc_defines.h
+@@ -536,7 +536,7 @@
+ #define IGC_PTM_CTRL_START_NOW	BIT(29) /* Start PTM Now */
+ #define IGC_PTM_CTRL_EN		BIT(30) /* Enable PTM */
+ #define IGC_PTM_CTRL_TRIG	BIT(31) /* PTM Cycle trigger */
+-#define IGC_PTM_CTRL_SHRT_CYC(usec)	(((usec) & 0x2f) << 2)
++#define IGC_PTM_CTRL_SHRT_CYC(usec)	(((usec) & 0x3f) << 2)
+ #define IGC_PTM_CTRL_PTM_TO(usec)	(((usec) & 0xff) << 8)
+ 
+ #define IGC_PTM_SHORT_CYC_DEFAULT	10  /* Default Short/interrupted cycle interval */
+diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+index 705325431dec3..5541e284cd3f0 100644
+--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
++++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+@@ -4005,9 +4005,10 @@ rx_frscfg:
+ 	if (link < 0)
+ 		return NIX_AF_ERR_RX_LINK_INVALID;
+ 
+-	nix_find_link_frs(rvu, req, pcifunc);
+ 
+ linkcfg:
++	nix_find_link_frs(rvu, req, pcifunc);
++
+ 	cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link));
+ 	cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16);
+ 	if (req->update_minlen)
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c
+index bd1a51a0a5408..f208a237d0b52 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c
+@@ -32,8 +32,8 @@ static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
+ 	MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8),
+ 	MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2),
+ 	MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6),
+-	MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 3),
+-	MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 20, 8),
++	MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 4),
++	MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 21, 8),
+ 	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4),
+ 	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4),
+ 	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4),
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
+index c968309657dd1..51eea1f0529c8 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
+@@ -517,11 +517,15 @@ static void mlxsw_pci_skb_cb_ts_set(struct mlxsw_pci *mlxsw_pci,
+ 				    struct sk_buff *skb,
+ 				    enum mlxsw_pci_cqe_v cqe_v, char *cqe)
+ {
++	u8 ts_type;
++
+ 	if (cqe_v != MLXSW_PCI_CQE_V2)
+ 		return;
+ 
+-	if (mlxsw_pci_cqe2_time_stamp_type_get(cqe) !=
+-	    MLXSW_PCI_CQE_TIME_STAMP_TYPE_UTC)
++	ts_type = mlxsw_pci_cqe2_time_stamp_type_get(cqe);
++
++	if (ts_type != MLXSW_PCI_CQE_TIME_STAMP_TYPE_UTC &&
++	    ts_type != MLXSW_PCI_CQE_TIME_STAMP_TYPE_MIRROR_UTC)
+ 		return;
+ 
+ 	mlxsw_skb_cb(skb)->cqe_ts.sec = mlxsw_pci_cqe2_time_stamp_sec_get(cqe);
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
+index 0777bed5bb1af..a34ff19c58bd2 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
++++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
+@@ -97,14 +97,6 @@ MLXSW_ITEM32(reg, sspr, m, 0x00, 31, 1);
+  */
+ MLXSW_ITEM32_LP(reg, sspr, 0x00, 16, 0x00, 12);
+ 
+-/* reg_sspr_sub_port
+- * Virtual port within the physical port.
+- * Should be set to 0 when virtual ports are not enabled on the port.
+- *
+- * Access: RW
+- */
+-MLXSW_ITEM32(reg, sspr, sub_port, 0x00, 8, 8);
+-
+ /* reg_sspr_system_port
+  * Unique identifier within the stacking domain that represents all the ports
+  * that are available in the system (external ports).
+@@ -120,7 +112,6 @@ static inline void mlxsw_reg_sspr_pack(char *payload, u16 local_port)
+ 	MLXSW_REG_ZERO(sspr, payload);
+ 	mlxsw_reg_sspr_m_set(payload, 1);
+ 	mlxsw_reg_sspr_local_port_set(payload, local_port);
+-	mlxsw_reg_sspr_sub_port_set(payload, 0);
+ 	mlxsw_reg_sspr_system_port_set(payload, local_port);
+ }
+ 
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum2_mr_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum2_mr_tcam.c
+index e4f4cded2b6f9..b1178b7a7f51a 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum2_mr_tcam.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum2_mr_tcam.c
+@@ -193,7 +193,7 @@ mlxsw_sp2_mr_tcam_rule_parse(struct mlxsw_sp_acl_rule *rule,
+ 				       key->vrid, GENMASK(7, 0));
+ 	mlxsw_sp_acl_rulei_keymask_u32(rulei,
+ 				       MLXSW_AFK_ELEMENT_VIRT_ROUTER_MSB,
+-				       key->vrid >> 8, GENMASK(2, 0));
++				       key->vrid >> 8, GENMASK(3, 0));
+ 	switch (key->proto) {
+ 	case MLXSW_SP_L3_PROTO_IPV4:
+ 		return mlxsw_sp2_mr_tcam_rule_parse4(rulei, key);
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c
+index 00c32320f8915..173808c096bab 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.c
+@@ -169,7 +169,7 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_2[] = {
+ 
+ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_4[] = {
+ 	MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_LSB, 0x04, 24, 8),
+-	MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_MSB, 0x00, 0, 3),
++	MLXSW_AFK_ELEMENT_INST_EXT_U32(VIRT_ROUTER_MSB, 0x00, 0, 3, 0, true),
+ };
+ 
+ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv6_0[] = {
+@@ -319,7 +319,7 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_mac_5b[] = {
+ 
+ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_4b[] = {
+ 	MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_LSB, 0x04, 13, 8),
+-	MLXSW_AFK_ELEMENT_INST_EXT_U32(VIRT_ROUTER_MSB, 0x04, 21, 4, 0, true),
++	MLXSW_AFK_ELEMENT_INST_U32(VIRT_ROUTER_MSB, 0x04, 21, 4),
+ };
+ 
+ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv6_2b[] = {
+diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
+index 796a38f9d7b24..cd16bc8bf154c 100644
+--- a/drivers/net/ipvlan/ipvlan_main.c
++++ b/drivers/net/ipvlan/ipvlan_main.c
+@@ -748,7 +748,8 @@ static int ipvlan_device_event(struct notifier_block *unused,
+ 
+ 		write_pnet(&port->pnet, newnet);
+ 
+-		ipvlan_migrate_l3s_hook(oldnet, newnet);
++		if (port->mode == IPVLAN_MODE_L3S)
++			ipvlan_migrate_l3s_hook(oldnet, newnet);
+ 		break;
+ 	}
+ 	case NETDEV_UNREGISTER:
+diff --git a/drivers/net/veth.c b/drivers/net/veth.c
+index a71786b3e7ba7..727b9278b9fe5 100644
+--- a/drivers/net/veth.c
++++ b/drivers/net/veth.c
+@@ -1716,10 +1716,7 @@ static int veth_newlink(struct net *src_net, struct net_device *dev,
+ 
+ 		nla_peer = data[VETH_INFO_PEER];
+ 		ifmp = nla_data(nla_peer);
+-		err = rtnl_nla_parse_ifla(peer_tb,
+-					  nla_data(nla_peer) + sizeof(struct ifinfomsg),
+-					  nla_len(nla_peer) - sizeof(struct ifinfomsg),
+-					  NULL);
++		err = rtnl_nla_parse_ifinfomsg(peer_tb, nla_peer, extack);
+ 		if (err < 0)
+ 			return err;
+ 
+diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c
+index cd3821a6444f0..4e436f2d13aeb 100644
+--- a/drivers/of/dynamic.c
++++ b/drivers/of/dynamic.c
+@@ -63,15 +63,14 @@ int of_reconfig_notifier_unregister(struct notifier_block *nb)
+ }
+ EXPORT_SYMBOL_GPL(of_reconfig_notifier_unregister);
+ 
+-#ifdef DEBUG
+-const char *action_names[] = {
++static const char *action_names[] = {
++	[0] = "INVALID",
+ 	[OF_RECONFIG_ATTACH_NODE] = "ATTACH_NODE",
+ 	[OF_RECONFIG_DETACH_NODE] = "DETACH_NODE",
+ 	[OF_RECONFIG_ADD_PROPERTY] = "ADD_PROPERTY",
+ 	[OF_RECONFIG_REMOVE_PROPERTY] = "REMOVE_PROPERTY",
+ 	[OF_RECONFIG_UPDATE_PROPERTY] = "UPDATE_PROPERTY",
+ };
+-#endif
+ 
+ int of_reconfig_notify(unsigned long action, struct of_reconfig_data *p)
+ {
+@@ -594,21 +593,9 @@ static int __of_changeset_entry_apply(struct of_changeset_entry *ce)
+ 		}
+ 
+ 		ret = __of_add_property(ce->np, ce->prop);
+-		if (ret) {
+-			pr_err("changeset: add_property failed @%pOF/%s\n",
+-				ce->np,
+-				ce->prop->name);
+-			break;
+-		}
+ 		break;
+ 	case OF_RECONFIG_REMOVE_PROPERTY:
+ 		ret = __of_remove_property(ce->np, ce->prop);
+-		if (ret) {
+-			pr_err("changeset: remove_property failed @%pOF/%s\n",
+-				ce->np,
+-				ce->prop->name);
+-			break;
+-		}
+ 		break;
+ 
+ 	case OF_RECONFIG_UPDATE_PROPERTY:
+@@ -622,20 +609,17 @@ static int __of_changeset_entry_apply(struct of_changeset_entry *ce)
+ 		}
+ 
+ 		ret = __of_update_property(ce->np, ce->prop, &old_prop);
+-		if (ret) {
+-			pr_err("changeset: update_property failed @%pOF/%s\n",
+-				ce->np,
+-				ce->prop->name);
+-			break;
+-		}
+ 		break;
+ 	default:
+ 		ret = -EINVAL;
+ 	}
+ 	raw_spin_unlock_irqrestore(&devtree_lock, flags);
+ 
+-	if (ret)
++	if (ret) {
++		pr_err("changeset: apply failed: %-15s %pOF:%s\n",
++		       action_names[ce->action], ce->np, ce->prop->name);
+ 		return ret;
++	}
+ 
+ 	switch (ce->action) {
+ 	case OF_RECONFIG_ATTACH_NODE:
+@@ -921,6 +905,9 @@ int of_changeset_action(struct of_changeset *ocs, unsigned long action,
+ 	if (!ce)
+ 		return -ENOMEM;
+ 
++	if (WARN_ON(action >= ARRAY_SIZE(action_names)))
++		return -EINVAL;
++
+ 	/* get a reference to the node */
+ 	ce->action = action;
+ 	ce->np = of_node_get(np);
+diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c
+index f26d2ba8a3715..68278340cecfe 100644
+--- a/drivers/of/kexec.c
++++ b/drivers/of/kexec.c
+@@ -184,7 +184,8 @@ int __init ima_free_kexec_buffer(void)
+ 	if (ret)
+ 		return ret;
+ 
+-	return memblock_phys_free(addr, size);
++	memblock_free_late(addr, size);
++	return 0;
+ }
+ #endif
+ 
+diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
+index b89ab5d9fea55..9be6ed47a1ce4 100644
+--- a/drivers/of/unittest.c
++++ b/drivers/of/unittest.c
+@@ -657,12 +657,12 @@ static void __init of_unittest_parse_phandle_with_args_map(void)
+ 	memset(&args, 0, sizeof(args));
+ 
+ 	EXPECT_BEGIN(KERN_INFO,
+-		     "OF: /testcase-data/phandle-tests/consumer-b: could not find phandle");
++		     "OF: /testcase-data/phandle-tests/consumer-b: could not find phandle 12345678");
+ 
+ 	rc = of_parse_phandle_with_args_map(np, "phandle-list-bad-phandle",
+ 					    "phandle", 0, &args);
+ 	EXPECT_END(KERN_INFO,
+-		   "OF: /testcase-data/phandle-tests/consumer-b: could not find phandle");
++		   "OF: /testcase-data/phandle-tests/consumer-b: could not find phandle 12345678");
+ 
+ 	unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
+ 
+diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c
+index 6efa3d8db9a56..ea0195337bab9 100644
+--- a/drivers/pci/hotplug/acpiphp_glue.c
++++ b/drivers/pci/hotplug/acpiphp_glue.c
+@@ -504,12 +504,15 @@ static void enable_slot(struct acpiphp_slot *slot, bool bridge)
+ 				if (pass && dev->subordinate) {
+ 					check_hotplug_bridge(slot, dev);
+ 					pcibios_resource_survey_bus(dev->subordinate);
+-					__pci_bus_size_bridges(dev->subordinate,
+-							       &add_list);
++					if (pci_is_root_bus(bus))
++						__pci_bus_size_bridges(dev->subordinate, &add_list);
+ 				}
+ 			}
+ 		}
+-		__pci_bus_assign_resources(bus, &add_list, NULL);
++		if (pci_is_root_bus(bus))
++			__pci_bus_assign_resources(bus, &add_list, NULL);
++		else
++			pci_assign_unassigned_bridge_resources(bus->self);
+ 	}
+ 
+ 	acpiphp_sanitize_bus(bus);
+diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
+index a8df77e80549c..be6838c252f09 100644
+--- a/drivers/pinctrl/pinctrl-amd.c
++++ b/drivers/pinctrl/pinctrl-amd.c
+@@ -862,6 +862,33 @@ static const struct pinconf_ops amd_pinconf_ops = {
+ 	.pin_config_group_set = amd_pinconf_group_set,
+ };
+ 
++static void amd_gpio_irq_init(struct amd_gpio *gpio_dev)
++{
++	struct pinctrl_desc *desc = gpio_dev->pctrl->desc;
++	unsigned long flags;
++	u32 pin_reg, mask;
++	int i;
++
++	mask = BIT(WAKE_CNTRL_OFF_S0I3) | BIT(WAKE_CNTRL_OFF_S3) |
++		BIT(WAKE_CNTRL_OFF_S4);
++
++	for (i = 0; i < desc->npins; i++) {
++		int pin = desc->pins[i].number;
++		const struct pin_desc *pd = pin_desc_get(gpio_dev->pctrl, pin);
++
++		if (!pd)
++			continue;
++
++		raw_spin_lock_irqsave(&gpio_dev->lock, flags);
++
++		pin_reg = readl(gpio_dev->base + pin * 4);
++		pin_reg &= ~mask;
++		writel(pin_reg, gpio_dev->base + pin * 4);
++
++		raw_spin_unlock_irqrestore(&gpio_dev->lock, flags);
++	}
++}
++
+ #ifdef CONFIG_PM_SLEEP
+ static bool amd_gpio_should_save(struct amd_gpio *gpio_dev, unsigned int pin)
+ {
+@@ -1099,6 +1126,9 @@ static int amd_gpio_probe(struct platform_device *pdev)
+ 		return PTR_ERR(gpio_dev->pctrl);
+ 	}
+ 
++	/* Disable and mask interrupts */
++	amd_gpio_irq_init(gpio_dev);
++
+ 	girq = &gpio_dev->gc.irq;
+ 	gpio_irq_chip_set_chip(girq, &amd_gpio_irqchip);
+ 	/* This will let us handle the parent IRQ in the driver */
+diff --git a/drivers/pinctrl/renesas/pinctrl-rza2.c b/drivers/pinctrl/renesas/pinctrl-rza2.c
+index c0a04f1ee994e..12126e30dc20f 100644
+--- a/drivers/pinctrl/renesas/pinctrl-rza2.c
++++ b/drivers/pinctrl/renesas/pinctrl-rza2.c
+@@ -14,6 +14,7 @@
+ #include <linux/gpio/driver.h>
+ #include <linux/io.h>
+ #include <linux/module.h>
++#include <linux/mutex.h>
+ #include <linux/of_device.h>
+ #include <linux/pinctrl/pinmux.h>
+ 
+@@ -46,6 +47,7 @@ struct rza2_pinctrl_priv {
+ 	struct pinctrl_dev *pctl;
+ 	struct pinctrl_gpio_range gpio_range;
+ 	int npins;
++	struct mutex mutex; /* serialize adding groups and functions */
+ };
+ 
+ #define RZA2_PDR(port)		(0x0000 + (port) * 2)	/* Direction 16-bit */
+@@ -358,10 +360,14 @@ static int rza2_dt_node_to_map(struct pinctrl_dev *pctldev,
+ 		psel_val[i] = MUX_FUNC(value);
+ 	}
+ 
++	mutex_lock(&priv->mutex);
++
+ 	/* Register a single pin group listing all the pins we read from DT */
+ 	gsel = pinctrl_generic_add_group(pctldev, np->name, pins, npins, NULL);
+-	if (gsel < 0)
+-		return gsel;
++	if (gsel < 0) {
++		ret = gsel;
++		goto unlock;
++	}
+ 
+ 	/*
+ 	 * Register a single group function where the 'data' is an array PSEL
+@@ -390,6 +396,8 @@ static int rza2_dt_node_to_map(struct pinctrl_dev *pctldev,
+ 	(*map)->data.mux.function = np->name;
+ 	*num_maps = 1;
+ 
++	mutex_unlock(&priv->mutex);
++
+ 	return 0;
+ 
+ remove_function:
+@@ -398,6 +406,9 @@ remove_function:
+ remove_group:
+ 	pinctrl_generic_remove_group(pctldev, gsel);
+ 
++unlock:
++	mutex_unlock(&priv->mutex);
++
+ 	dev_err(priv->dev, "Unable to parse DT node %s\n", np->name);
+ 
+ 	return ret;
+@@ -473,6 +484,8 @@ static int rza2_pinctrl_probe(struct platform_device *pdev)
+ 	if (IS_ERR(priv->base))
+ 		return PTR_ERR(priv->base);
+ 
++	mutex_init(&priv->mutex);
++
+ 	platform_set_drvdata(pdev, priv);
+ 
+ 	priv->npins = (int)(uintptr_t)of_device_get_match_data(&pdev->dev) *
+diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+index fd11d28e5a1e4..2a617832a7e60 100644
+--- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c
++++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+@@ -11,6 +11,7 @@
+ #include <linux/io.h>
+ #include <linux/interrupt.h>
+ #include <linux/module.h>
++#include <linux/mutex.h>
+ #include <linux/of_device.h>
+ #include <linux/of_irq.h>
+ #include <linux/pinctrl/pinconf-generic.h>
+@@ -146,10 +147,11 @@ struct rzg2l_pinctrl {
+ 	struct gpio_chip		gpio_chip;
+ 	struct pinctrl_gpio_range	gpio_range;
+ 	DECLARE_BITMAP(tint_slot, RZG2L_TINT_MAX_INTERRUPT);
+-	spinlock_t			bitmap_lock;
++	spinlock_t			bitmap_lock; /* protect tint_slot bitmap */
+ 	unsigned int			hwirq[RZG2L_TINT_MAX_INTERRUPT];
+ 
+-	spinlock_t			lock;
++	spinlock_t			lock; /* lock read/write registers */
++	struct mutex			mutex; /* serialize adding groups and functions */
+ };
+ 
+ static const unsigned int iolh_groupa_mA[] = { 2, 4, 8, 12 };
+@@ -359,11 +361,13 @@ static int rzg2l_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+ 		name = np->name;
+ 	}
+ 
++	mutex_lock(&pctrl->mutex);
++
+ 	/* Register a single pin group listing all the pins we read from DT */
+ 	gsel = pinctrl_generic_add_group(pctldev, name, pins, num_pinmux, NULL);
+ 	if (gsel < 0) {
+ 		ret = gsel;
+-		goto done;
++		goto unlock;
+ 	}
+ 
+ 	/*
+@@ -377,6 +381,8 @@ static int rzg2l_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+ 		goto remove_group;
+ 	}
+ 
++	mutex_unlock(&pctrl->mutex);
++
+ 	maps[idx].type = PIN_MAP_TYPE_MUX_GROUP;
+ 	maps[idx].data.mux.group = name;
+ 	maps[idx].data.mux.function = name;
+@@ -388,6 +394,8 @@ static int rzg2l_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+ 
+ remove_group:
+ 	pinctrl_generic_remove_group(pctldev, gsel);
++unlock:
++	mutex_unlock(&pctrl->mutex);
+ done:
+ 	*index = idx;
+ 	kfree(configs);
+@@ -1501,6 +1509,7 @@ static int rzg2l_pinctrl_probe(struct platform_device *pdev)
+ 
+ 	spin_lock_init(&pctrl->lock);
+ 	spin_lock_init(&pctrl->bitmap_lock);
++	mutex_init(&pctrl->mutex);
+ 
+ 	platform_set_drvdata(pdev, pctrl);
+ 
+diff --git a/drivers/pinctrl/renesas/pinctrl-rzv2m.c b/drivers/pinctrl/renesas/pinctrl-rzv2m.c
+index 35f382b055e83..2858800288bb7 100644
+--- a/drivers/pinctrl/renesas/pinctrl-rzv2m.c
++++ b/drivers/pinctrl/renesas/pinctrl-rzv2m.c
+@@ -14,6 +14,7 @@
+ #include <linux/gpio/driver.h>
+ #include <linux/io.h>
+ #include <linux/module.h>
++#include <linux/mutex.h>
+ #include <linux/of_device.h>
+ #include <linux/pinctrl/pinconf-generic.h>
+ #include <linux/pinctrl/pinconf.h>
+@@ -121,7 +122,8 @@ struct rzv2m_pinctrl {
+ 	struct gpio_chip		gpio_chip;
+ 	struct pinctrl_gpio_range	gpio_range;
+ 
+-	spinlock_t			lock;
++	spinlock_t			lock; /* lock read/write registers */
++	struct mutex			mutex; /* serialize adding groups and functions */
+ };
+ 
+ static const unsigned int drv_1_8V_group2_uA[] = { 1800, 3800, 7800, 11000 };
+@@ -320,11 +322,13 @@ static int rzv2m_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+ 		name = np->name;
+ 	}
+ 
++	mutex_lock(&pctrl->mutex);
++
+ 	/* Register a single pin group listing all the pins we read from DT */
+ 	gsel = pinctrl_generic_add_group(pctldev, name, pins, num_pinmux, NULL);
+ 	if (gsel < 0) {
+ 		ret = gsel;
+-		goto done;
++		goto unlock;
+ 	}
+ 
+ 	/*
+@@ -338,6 +342,8 @@ static int rzv2m_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+ 		goto remove_group;
+ 	}
+ 
++	mutex_unlock(&pctrl->mutex);
++
+ 	maps[idx].type = PIN_MAP_TYPE_MUX_GROUP;
+ 	maps[idx].data.mux.group = name;
+ 	maps[idx].data.mux.function = name;
+@@ -349,6 +355,8 @@ static int rzv2m_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+ 
+ remove_group:
+ 	pinctrl_generic_remove_group(pctldev, gsel);
++unlock:
++	mutex_unlock(&pctrl->mutex);
+ done:
+ 	*index = idx;
+ 	kfree(configs);
+@@ -1070,6 +1078,7 @@ static int rzv2m_pinctrl_probe(struct platform_device *pdev)
+ 	}
+ 
+ 	spin_lock_init(&pctrl->lock);
++	mutex_init(&pctrl->mutex);
+ 
+ 	platform_set_drvdata(pdev, pctrl);
+ 
+diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c
+index bd38c7dcae347..de03b8889e9d3 100644
+--- a/drivers/platform/x86/ideapad-laptop.c
++++ b/drivers/platform/x86/ideapad-laptop.c
+@@ -1176,6 +1176,11 @@ static const struct key_entry ideapad_keymap[] = {
+ 	{ KE_IGNORE,	0x03 | IDEAPAD_WMI_KEY },
+ 	/* Customizable Lenovo Hotkey ("star" with 'S' inside) */
+ 	{ KE_KEY,	0x01 | IDEAPAD_WMI_KEY, { KEY_FAVORITES } },
++	{ KE_KEY,	0x04 | IDEAPAD_WMI_KEY, { KEY_SELECTIVE_SCREENSHOT } },
++	/* Lenovo Support */
++	{ KE_KEY,	0x07 | IDEAPAD_WMI_KEY, { KEY_HELP } },
++	{ KE_KEY,	0x0e | IDEAPAD_WMI_KEY, { KEY_PICKUP_PHONE } },
++	{ KE_KEY,	0x0f | IDEAPAD_WMI_KEY, { KEY_HANGUP_PHONE } },
+ 	/* Dark mode toggle */
+ 	{ KE_KEY,	0x13 | IDEAPAD_WMI_KEY, { KEY_PROG1 } },
+ 	/* Sound profile switch */
+diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c
+index f99a9ef42116f..84e3ad290f6ba 100644
+--- a/drivers/s390/crypto/zcrypt_msgtype6.c
++++ b/drivers/s390/crypto/zcrypt_msgtype6.c
+@@ -926,8 +926,7 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq,
+ 		.type = TYPE82_RSP_CODE,
+ 		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+ 	};
+-	struct response_type *resp_type =
+-		(struct response_type *)msg->private;
++	struct response_type *resp_type = msg->private;
+ 	struct type86x_reply *t86r;
+ 	int len;
+ 
+@@ -982,8 +981,7 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq,
+ 		.type = TYPE82_RSP_CODE,
+ 		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+ 	};
+-	struct response_type *resp_type =
+-		(struct response_type *)msg->private;
++	struct response_type *resp_type = msg->private;
+ 	struct type86_ep11_reply *t86r;
+ 	int len;
+ 
+@@ -1156,23 +1154,36 @@ static long zcrypt_msgtype6_send_cprb(bool userspace, struct zcrypt_queue *zq,
+ 				      struct ica_xcRB *xcrb,
+ 				      struct ap_message *ap_msg)
+ {
+-	int rc;
+-	struct response_type *rtype = (struct response_type *)(ap_msg->private);
++	struct response_type *rtype = ap_msg->private;
+ 	struct {
+ 		struct type6_hdr hdr;
+ 		struct CPRBX cprbx;
+ 		/* ... more data blocks ... */
+ 	} __packed * msg = ap_msg->msg;
+-
+-	/*
+-	 * Set the queue's reply buffer length minus 128 byte padding
+-	 * as reply limit for the card firmware.
+-	 */
+-	msg->hdr.fromcardlen1 = min_t(unsigned int, msg->hdr.fromcardlen1,
+-				      zq->reply.bufsize - 128);
+-	if (msg->hdr.fromcardlen2)
+-		msg->hdr.fromcardlen2 =
+-			zq->reply.bufsize - msg->hdr.fromcardlen1 - 128;
++	unsigned int max_payload_size;
++	int rc, delta;
++
++	/* calculate maximum payload for this card and msg type */
++	max_payload_size = zq->reply.bufsize - sizeof(struct type86_fmt2_msg);
++
++	/* limit each of the two from fields to the maximum payload size */
++	msg->hdr.fromcardlen1 = min(msg->hdr.fromcardlen1, max_payload_size);
++	msg->hdr.fromcardlen2 = min(msg->hdr.fromcardlen2, max_payload_size);
++
++	/* calculate delta if the sum of both exceeds max payload size */
++	delta = msg->hdr.fromcardlen1 + msg->hdr.fromcardlen2
++		- max_payload_size;
++	if (delta > 0) {
++		/*
++		 * Sum exceeds maximum payload size, prune fromcardlen1
++		 * (always trust fromcardlen2)
++		 */
++		if (delta > msg->hdr.fromcardlen1) {
++			rc = -EINVAL;
++			goto out;
++		}
++		msg->hdr.fromcardlen1 -= delta;
++	}
+ 
+ 	init_completion(&rtype->work);
+ 	rc = ap_queue_message(zq->queue, ap_msg);
+@@ -1243,7 +1254,7 @@ static long zcrypt_msgtype6_send_ep11_cprb(bool userspace, struct zcrypt_queue *
+ {
+ 	int rc;
+ 	unsigned int lfmt;
+-	struct response_type *rtype = (struct response_type *)(ap_msg->private);
++	struct response_type *rtype = ap_msg->private;
+ 	struct {
+ 		struct type6_hdr hdr;
+ 		struct ep11_cprb cprbx;
+@@ -1365,7 +1376,7 @@ static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq,
+ 		short int verb_length;
+ 		short int key_length;
+ 	} __packed * msg = ap_msg->msg;
+-	struct response_type *rtype = (struct response_type *)(ap_msg->private);
++	struct response_type *rtype = ap_msg->private;
+ 	int rc;
+ 
+ 	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
+diff --git a/drivers/scsi/raid_class.c b/drivers/scsi/raid_class.c
+index 711252e52d8e1..95a86e0dfd77a 100644
+--- a/drivers/scsi/raid_class.c
++++ b/drivers/scsi/raid_class.c
+@@ -209,54 +209,6 @@ raid_attr_ro_state(level);
+ raid_attr_ro_fn(resync);
+ raid_attr_ro_state_fn(state);
+ 
+-static void raid_component_release(struct device *dev)
+-{
+-	struct raid_component *rc =
+-		container_of(dev, struct raid_component, dev);
+-	dev_printk(KERN_ERR, rc->dev.parent, "COMPONENT RELEASE\n");
+-	put_device(rc->dev.parent);
+-	kfree(rc);
+-}
+-
+-int raid_component_add(struct raid_template *r,struct device *raid_dev,
+-		       struct device *component_dev)
+-{
+-	struct device *cdev =
+-		attribute_container_find_class_device(&r->raid_attrs.ac,
+-						      raid_dev);
+-	struct raid_component *rc;
+-	struct raid_data *rd = dev_get_drvdata(cdev);
+-	int err;
+-
+-	rc = kzalloc(sizeof(*rc), GFP_KERNEL);
+-	if (!rc)
+-		return -ENOMEM;
+-
+-	INIT_LIST_HEAD(&rc->node);
+-	device_initialize(&rc->dev);
+-	rc->dev.release = raid_component_release;
+-	rc->dev.parent = get_device(component_dev);
+-	rc->num = rd->component_count++;
+-
+-	dev_set_name(&rc->dev, "component-%d", rc->num);
+-	list_add_tail(&rc->node, &rd->component_list);
+-	rc->dev.class = &raid_class.class;
+-	err = device_add(&rc->dev);
+-	if (err)
+-		goto err_out;
+-
+-	return 0;
+-
+-err_out:
+-	put_device(&rc->dev);
+-	list_del(&rc->node);
+-	rd->component_count--;
+-	put_device(component_dev);
+-	kfree(rc);
+-	return err;
+-}
+-EXPORT_SYMBOL(raid_component_add);
+-
+ struct raid_template *
+ raid_class_attach(struct raid_function_template *ft)
+ {
+diff --git a/drivers/scsi/snic/snic_disc.c b/drivers/scsi/snic/snic_disc.c
+index cd27562ec922e..6c529b37f3b46 100644
+--- a/drivers/scsi/snic/snic_disc.c
++++ b/drivers/scsi/snic/snic_disc.c
+@@ -303,12 +303,11 @@ snic_tgt_create(struct snic *snic, struct snic_tgt_id *tgtid)
+ 			      "Snic Tgt: device_add, with err = %d\n",
+ 			      ret);
+ 
+-		put_device(&tgt->dev);
+ 		put_device(&snic->shost->shost_gendev);
+ 		spin_lock_irqsave(snic->shost->host_lock, flags);
+ 		list_del(&tgt->list);
+ 		spin_unlock_irqrestore(snic->shost->host_lock, flags);
+-		kfree(tgt);
++		put_device(&tgt->dev);
+ 		tgt = NULL;
+ 
+ 		return tgt;
+diff --git a/drivers/thunderbolt/tmu.c b/drivers/thunderbolt/tmu.c
+index 626aca3124b1c..d9544600b3867 100644
+--- a/drivers/thunderbolt/tmu.c
++++ b/drivers/thunderbolt/tmu.c
+@@ -415,7 +415,8 @@ int tb_switch_tmu_disable(struct tb_switch *sw)
+ 		 * uni-directional mode and we don't want to change it's TMU
+ 		 * mode.
+ 		 */
+-		tb_switch_tmu_rate_write(sw, TB_SWITCH_TMU_RATE_OFF);
++		ret = tb_switch_tmu_rate_write(sw, TB_SWITCH_TMU_RATE_OFF);
++			return ret;
+ 
+ 		tb_port_tmu_time_sync_disable(up);
+ 		ret = tb_port_tmu_time_sync_disable(down);
+diff --git a/drivers/video/aperture.c b/drivers/video/aperture.c
+index 5c94abdb1ad6d..3e4a1f55f51b3 100644
+--- a/drivers/video/aperture.c
++++ b/drivers/video/aperture.c
+@@ -298,14 +298,6 @@ int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t si
+ 
+ 	aperture_detach_devices(base, size);
+ 
+-	/*
+-	 * If this is the primary adapter, there could be a VGA device
+-	 * that consumes the VGA framebuffer I/O range. Remove this device
+-	 * as well.
+-	 */
+-	if (primary)
+-		aperture_detach_devices(VGA_FB_PHYS_BASE, VGA_FB_PHYS_SIZE);
+-
+ 	return 0;
+ }
+ EXPORT_SYMBOL(aperture_remove_conflicting_devices);
+@@ -344,13 +336,22 @@ int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *na
+ 		aperture_detach_devices(base, size);
+ 	}
+ 
+-	/*
+-	 * WARNING: Apparently we must kick fbdev drivers before vgacon,
+-	 * otherwise the vga fbdev driver falls over.
+-	 */
+-	ret = vga_remove_vgacon(pdev);
+-	if (ret)
+-		return ret;
++	if (primary) {
++		/*
++		 * If this is the primary adapter, there could be a VGA device
++		 * that consumes the VGA framebuffer I/O range. Remove this
++		 * device as well.
++		 */
++		aperture_detach_devices(VGA_FB_PHYS_BASE, VGA_FB_PHYS_SIZE);
++
++		/*
++		 * WARNING: Apparently we must kick fbdev drivers before vgacon,
++		 * otherwise the vga fbdev driver falls over.
++		 */
++		ret = vga_remove_vgacon(pdev);
++		if (ret)
++			return ret;
++	}
+ 
+ 	return 0;
+ 
+diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c
+index 8b28c9bddd974..50c384ce28837 100644
+--- a/drivers/video/fbdev/aty/radeon_base.c
++++ b/drivers/video/fbdev/aty/radeon_base.c
+@@ -2238,14 +2238,6 @@ static const struct bin_attribute edid2_attr = {
+ 	.read	= radeon_show_edid2,
+ };
+ 
+-static int radeon_kick_out_firmware_fb(struct pci_dev *pdev)
+-{
+-	resource_size_t base = pci_resource_start(pdev, 0);
+-	resource_size_t size = pci_resource_len(pdev, 0);
+-
+-	return aperture_remove_conflicting_devices(base, size, false, KBUILD_MODNAME);
+-}
+-
+ static int radeonfb_pci_register(struct pci_dev *pdev,
+ 				 const struct pci_device_id *ent)
+ {
+@@ -2296,7 +2288,7 @@ static int radeonfb_pci_register(struct pci_dev *pdev,
+ 	rinfo->fb_base_phys = pci_resource_start (pdev, 0);
+ 	rinfo->mmio_base_phys = pci_resource_start (pdev, 2);
+ 
+-	ret = radeon_kick_out_firmware_fb(pdev);
++	ret = aperture_remove_conflicting_pci_devices(pdev, KBUILD_MODNAME);
+ 	if (ret)
+ 		goto err_release_fb;
+ 
+diff --git a/fs/attr.c b/fs/attr.c
+index b45f30e516fad..9b9a70e0cc54f 100644
+--- a/fs/attr.c
++++ b/fs/attr.c
+@@ -47,6 +47,7 @@ int setattr_should_drop_sgid(struct user_namespace *mnt_userns,
+ 		return ATTR_KILL_SGID;
+ 	return 0;
+ }
++EXPORT_SYMBOL(setattr_should_drop_sgid);
+ 
+ /**
+  * setattr_should_drop_suidgid - determine whether the set{g,u}id bit needs to
+diff --git a/fs/internal.h b/fs/internal.h
+index 46caa33373a48..42df013f7fe76 100644
+--- a/fs/internal.h
++++ b/fs/internal.h
+@@ -242,5 +242,3 @@ ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *po
+ /*
+  * fs/attr.c
+  */
+-int setattr_should_drop_sgid(struct user_namespace *mnt_userns,
+-			     const struct inode *inode);
+diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c
+index c4e0da6db7195..9ec91017a7f3c 100644
+--- a/fs/jbd2/checkpoint.c
++++ b/fs/jbd2/checkpoint.c
+@@ -27,7 +27,7 @@
+  *
+  * Called with j_list_lock held.
+  */
+-static inline void __buffer_unlink_first(struct journal_head *jh)
++static inline void __buffer_unlink(struct journal_head *jh)
+ {
+ 	transaction_t *transaction = jh->b_cp_transaction;
+ 
+@@ -40,23 +40,6 @@ static inline void __buffer_unlink_first(struct journal_head *jh)
+ 	}
+ }
+ 
+-/*
+- * Unlink a buffer from a transaction checkpoint(io) list.
+- *
+- * Called with j_list_lock held.
+- */
+-static inline void __buffer_unlink(struct journal_head *jh)
+-{
+-	transaction_t *transaction = jh->b_cp_transaction;
+-
+-	__buffer_unlink_first(jh);
+-	if (transaction->t_checkpoint_io_list == jh) {
+-		transaction->t_checkpoint_io_list = jh->b_cpnext;
+-		if (transaction->t_checkpoint_io_list == jh)
+-			transaction->t_checkpoint_io_list = NULL;
+-	}
+-}
+-
+ /*
+  * Check a checkpoint buffer could be release or not.
+  *
+@@ -366,50 +349,10 @@ int jbd2_cleanup_journal_tail(journal_t *journal)
+ 
+ /* Checkpoint list management */
+ 
+-/*
+- * journal_clean_one_cp_list
+- *
+- * Find all the written-back checkpoint buffers in the given list and
+- * release them. If 'destroy' is set, clean all buffers unconditionally.
+- *
+- * Called with j_list_lock held.
+- * Returns 1 if we freed the transaction, 0 otherwise.
+- */
+-static int journal_clean_one_cp_list(struct journal_head *jh, bool destroy)
+-{
+-	struct journal_head *last_jh;
+-	struct journal_head *next_jh = jh;
+-
+-	if (!jh)
+-		return 0;
+-
+-	last_jh = jh->b_cpprev;
+-	do {
+-		jh = next_jh;
+-		next_jh = jh->b_cpnext;
+-
+-		if (!destroy && __cp_buffer_busy(jh))
+-			return 0;
+-
+-		if (__jbd2_journal_remove_checkpoint(jh))
+-			return 1;
+-		/*
+-		 * This function only frees up some memory
+-		 * if possible so we dont have an obligation
+-		 * to finish processing. Bail out if preemption
+-		 * requested:
+-		 */
+-		if (need_resched())
+-			return 0;
+-	} while (jh != last_jh);
+-
+-	return 0;
+-}
+-
+ /*
+  * journal_shrink_one_cp_list
+  *
+- * Find 'nr_to_scan' written-back checkpoint buffers in the given list
++ * Find all the written-back checkpoint buffers in the given list
+  * and try to release them. If the whole transaction is released, set
+  * the 'released' parameter. Return the number of released checkpointed
+  * buffers.
+@@ -417,15 +360,15 @@ static int journal_clean_one_cp_list(struct journal_head *jh, bool destroy)
+  * Called with j_list_lock held.
+  */
+ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
+-						unsigned long *nr_to_scan,
+-						bool *released)
++						bool destroy, bool *released)
+ {
+ 	struct journal_head *last_jh;
+ 	struct journal_head *next_jh = jh;
+ 	unsigned long nr_freed = 0;
+ 	int ret;
+ 
+-	if (!jh || *nr_to_scan == 0)
++	*released = false;
++	if (!jh)
+ 		return 0;
+ 
+ 	last_jh = jh->b_cpprev;
+@@ -433,12 +376,15 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
+ 		jh = next_jh;
+ 		next_jh = jh->b_cpnext;
+ 
+-		(*nr_to_scan)--;
+-		if (__cp_buffer_busy(jh))
+-			continue;
++		if (destroy) {
++			ret = __jbd2_journal_remove_checkpoint(jh);
++		} else {
++			ret = jbd2_journal_try_remove_checkpoint(jh);
++			if (ret < 0)
++				continue;
++		}
+ 
+ 		nr_freed++;
+-		ret = __jbd2_journal_remove_checkpoint(jh);
+ 		if (ret) {
+ 			*released = true;
+ 			break;
+@@ -446,7 +392,7 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh,
+ 
+ 		if (need_resched())
+ 			break;
+-	} while (jh != last_jh && *nr_to_scan);
++	} while (jh != last_jh);
+ 
+ 	return nr_freed;
+ }
+@@ -464,11 +410,11 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal,
+ 						  unsigned long *nr_to_scan)
+ {
+ 	transaction_t *transaction, *last_transaction, *next_transaction;
+-	bool released;
++	bool __maybe_unused released;
+ 	tid_t first_tid = 0, last_tid = 0, next_tid = 0;
+ 	tid_t tid = 0;
+ 	unsigned long nr_freed = 0;
+-	unsigned long nr_scanned = *nr_to_scan;
++	unsigned long freed;
+ 
+ again:
+ 	spin_lock(&journal->j_list_lock);
+@@ -497,19 +443,11 @@ again:
+ 		transaction = next_transaction;
+ 		next_transaction = transaction->t_cpnext;
+ 		tid = transaction->t_tid;
+-		released = false;
+ 
+-		nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list,
+-						       nr_to_scan, &released);
+-		if (*nr_to_scan == 0)
+-			break;
+-		if (need_resched() || spin_needbreak(&journal->j_list_lock))
+-			break;
+-		if (released)
+-			continue;
+-
+-		nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_io_list,
+-						       nr_to_scan, &released);
++		freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list,
++						   false, &released);
++		nr_freed += freed;
++		(*nr_to_scan) -= min(*nr_to_scan, freed);
+ 		if (*nr_to_scan == 0)
+ 			break;
+ 		if (need_resched() || spin_needbreak(&journal->j_list_lock))
+@@ -530,9 +468,8 @@ again:
+ 	if (*nr_to_scan && next_tid)
+ 		goto again;
+ out:
+-	nr_scanned -= *nr_to_scan;
+ 	trace_jbd2_shrink_checkpoint_list(journal, first_tid, tid, last_tid,
+-					  nr_freed, nr_scanned, next_tid);
++					  nr_freed, next_tid);
+ 
+ 	return nr_freed;
+ }
+@@ -548,7 +485,7 @@ out:
+ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, bool destroy)
+ {
+ 	transaction_t *transaction, *last_transaction, *next_transaction;
+-	int ret;
++	bool released;
+ 
+ 	transaction = journal->j_checkpoint_transactions;
+ 	if (!transaction)
+@@ -559,8 +496,8 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, bool destroy)
+ 	do {
+ 		transaction = next_transaction;
+ 		next_transaction = transaction->t_cpnext;
+-		ret = journal_clean_one_cp_list(transaction->t_checkpoint_list,
+-						destroy);
++		journal_shrink_one_cp_list(transaction->t_checkpoint_list,
++					   destroy, &released);
+ 		/*
+ 		 * This function only frees up some memory if possible so we
+ 		 * dont have an obligation to finish processing. Bail out if
+@@ -568,23 +505,12 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, bool destroy)
+ 		 */
+ 		if (need_resched())
+ 			return;
+-		if (ret)
+-			continue;
+-		/*
+-		 * It is essential that we are as careful as in the case of
+-		 * t_checkpoint_list with removing the buffer from the list as
+-		 * we can possibly see not yet submitted buffers on io_list
+-		 */
+-		ret = journal_clean_one_cp_list(transaction->
+-				t_checkpoint_io_list, destroy);
+-		if (need_resched())
+-			return;
+ 		/*
+ 		 * Stop scanning if we couldn't free the transaction. This
+ 		 * avoids pointless scanning of transactions which still
+ 		 * weren't checkpointed.
+ 		 */
+-		if (!ret)
++		if (!released)
+ 			return;
+ 	} while (transaction != last_transaction);
+ }
+@@ -663,7 +589,7 @@ int __jbd2_journal_remove_checkpoint(struct journal_head *jh)
+ 	jbd2_journal_put_journal_head(jh);
+ 
+ 	/* Is this transaction empty? */
+-	if (transaction->t_checkpoint_list || transaction->t_checkpoint_io_list)
++	if (transaction->t_checkpoint_list)
+ 		return 0;
+ 
+ 	/*
+@@ -694,6 +620,34 @@ int __jbd2_journal_remove_checkpoint(struct journal_head *jh)
+ 	return 1;
+ }
+ 
++/*
++ * Check the checkpoint buffer and try to remove it from the checkpoint
++ * list if it's clean. Returns -EBUSY if it is not clean, returns 1 if
++ * it frees the transaction, 0 otherwise.
++ *
++ * This function is called with j_list_lock held.
++ */
++int jbd2_journal_try_remove_checkpoint(struct journal_head *jh)
++{
++	struct buffer_head *bh = jh2bh(jh);
++
++	if (!trylock_buffer(bh))
++		return -EBUSY;
++	if (buffer_dirty(bh)) {
++		unlock_buffer(bh);
++		return -EBUSY;
++	}
++	unlock_buffer(bh);
++
++	/*
++	 * Buffer is clean and the IO has finished (we held the buffer
++	 * lock) so the checkpoint is done. We can safely remove the
++	 * buffer from this transaction.
++	 */
++	JBUFFER_TRACE(jh, "remove from checkpoint list");
++	return __jbd2_journal_remove_checkpoint(jh);
++}
++
+ /*
+  * journal_insert_checkpoint: put a committed buffer onto a checkpoint
+  * list so that we know when it is safe to clean the transaction out of
+@@ -755,7 +709,6 @@ void __jbd2_journal_drop_transaction(journal_t *journal, transaction_t *transact
+ 	J_ASSERT(transaction->t_forget == NULL);
+ 	J_ASSERT(transaction->t_shadow_list == NULL);
+ 	J_ASSERT(transaction->t_checkpoint_list == NULL);
+-	J_ASSERT(transaction->t_checkpoint_io_list == NULL);
+ 	J_ASSERT(atomic_read(&transaction->t_updates) == 0);
+ 	J_ASSERT(journal->j_committing_transaction != transaction);
+ 	J_ASSERT(journal->j_running_transaction != transaction);
+diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c
+index 885a7a6cc53e6..f1d9db6686e31 100644
+--- a/fs/jbd2/commit.c
++++ b/fs/jbd2/commit.c
+@@ -1171,8 +1171,7 @@ restart_loop:
+ 	spin_lock(&journal->j_list_lock);
+ 	commit_transaction->t_state = T_FINISHED;
+ 	/* Check if the transaction can be dropped now that we are finished */
+-	if (commit_transaction->t_checkpoint_list == NULL &&
+-	    commit_transaction->t_checkpoint_io_list == NULL) {
++	if (commit_transaction->t_checkpoint_list == NULL) {
+ 		__jbd2_journal_drop_transaction(journal, commit_transaction);
+ 		jbd2_journal_free_transaction(commit_transaction);
+ 	}
+diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c
+index 18611241f4513..6ef5022949c46 100644
+--- a/fs/jbd2/transaction.c
++++ b/fs/jbd2/transaction.c
+@@ -1784,8 +1784,7 @@ int jbd2_journal_forget(handle_t *handle, struct buffer_head *bh)
+ 		 * Otherwise, if the buffer has been written to disk,
+ 		 * it is safe to remove the checkpoint and drop it.
+ 		 */
+-		if (!buffer_dirty(bh)) {
+-			__jbd2_journal_remove_checkpoint(jh);
++		if (jbd2_journal_try_remove_checkpoint(jh) >= 0) {
+ 			spin_unlock(&journal->j_list_lock);
+ 			goto drop;
+ 		}
+@@ -2112,20 +2111,14 @@ __journal_try_to_free_buffer(journal_t *journal, struct buffer_head *bh)
+ 
+ 	jh = bh2jh(bh);
+ 
+-	if (buffer_locked(bh) || buffer_dirty(bh))
+-		goto out;
+-
+ 	if (jh->b_next_transaction != NULL || jh->b_transaction != NULL)
+-		goto out;
++		return;
+ 
+ 	spin_lock(&journal->j_list_lock);
+-	if (jh->b_cp_transaction != NULL) {
+-		/* written-back checkpointed metadata buffer */
+-		JBUFFER_TRACE(jh, "remove from checkpoint list");
+-		__jbd2_journal_remove_checkpoint(jh);
+-	}
++	/* Remove written-back checkpointed metadata buffer */
++	if (jh->b_cp_transaction != NULL)
++		jbd2_journal_try_remove_checkpoint(jh);
+ 	spin_unlock(&journal->j_list_lock);
+-out:
+ 	return;
+ }
+ 
+diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
+index 1707f46b1335c..cf34d0c309459 100644
+--- a/fs/nfs/direct.c
++++ b/fs/nfs/direct.c
+@@ -474,20 +474,26 @@ out:
+ 	return result;
+ }
+ 
+-static void
+-nfs_direct_join_group(struct list_head *list, struct inode *inode)
++static void nfs_direct_join_group(struct list_head *list, struct inode *inode)
+ {
+-	struct nfs_page *req, *next;
++	struct nfs_page *req, *subreq;
+ 
+ 	list_for_each_entry(req, list, wb_list) {
+-		if (req->wb_head != req || req->wb_this_page == req)
++		if (req->wb_head != req)
+ 			continue;
+-		for (next = req->wb_this_page;
+-				next != req->wb_head;
+-				next = next->wb_this_page) {
+-			nfs_list_remove_request(next);
+-			nfs_release_request(next);
+-		}
++		subreq = req->wb_this_page;
++		if (subreq == req)
++			continue;
++		do {
++			/*
++			 * Remove subrequests from this list before freeing
++			 * them in the call to nfs_join_page_group().
++			 */
++			if (!list_empty(&subreq->wb_list)) {
++				nfs_list_remove_request(subreq);
++				nfs_release_request(subreq);
++			}
++		} while ((subreq = subreq->wb_this_page) != req);
+ 		nfs_join_page_group(req, inode);
+ 	}
+ }
+diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
+index 6b2cfa59a1a2b..e0c1fb98f907a 100644
+--- a/fs/nfs/inode.c
++++ b/fs/nfs/inode.c
+@@ -717,9 +717,7 @@ void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr,
+ 		if ((attr->ia_valid & ATTR_KILL_SUID) != 0 &&
+ 		    inode->i_mode & S_ISUID)
+ 			inode->i_mode &= ~S_ISUID;
+-		if ((attr->ia_valid & ATTR_KILL_SGID) != 0 &&
+-		    (inode->i_mode & (S_ISGID | S_IXGRP)) ==
+-		     (S_ISGID | S_IXGRP))
++		if (setattr_should_drop_sgid(&init_user_ns, inode))
+ 			inode->i_mode &= ~S_ISGID;
+ 		if ((attr->ia_valid & ATTR_MODE) != 0) {
+ 			int mode = attr->ia_mode & S_IALLUGO;
+diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
+index ecb428512fe1a..7c33bba179d2f 100644
+--- a/fs/nfs/nfs42proc.c
++++ b/fs/nfs/nfs42proc.c
+@@ -1359,7 +1359,6 @@ ssize_t nfs42_proc_getxattr(struct inode *inode, const char *name,
+ 	for (i = 0; i < np; i++) {
+ 		pages[i] = alloc_page(GFP_KERNEL);
+ 		if (!pages[i]) {
+-			np = i + 1;
+ 			err = -ENOMEM;
+ 			goto out;
+ 		}
+@@ -1383,8 +1382,8 @@ ssize_t nfs42_proc_getxattr(struct inode *inode, const char *name,
+ 	} while (exception.retry);
+ 
+ out:
+-	while (--np >= 0)
+-		__free_page(pages[np]);
++	while (--i >= 0)
++		__free_page(pages[i]);
+ 	kfree(pages);
+ 
+ 	return err;
+diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
+index 177cb7b089b9a..1044305e77996 100644
+--- a/fs/nfs/nfs4proc.c
++++ b/fs/nfs/nfs4proc.c
+@@ -5995,9 +5995,8 @@ static ssize_t __nfs4_get_acl_uncached(struct inode *inode, void *buf,
+ out_ok:
+ 	ret = res.acl_len;
+ out_free:
+-	for (i = 0; i < npages; i++)
+-		if (pages[i])
+-			__free_page(pages[i]);
++	while (--i >= 0)
++		__free_page(pages[i]);
+ 	if (res.acl_scratch)
+ 		__free_page(res.acl_scratch);
+ 	kfree(pages);
+@@ -7171,8 +7170,15 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata)
+ 		} else if (!nfs4_update_lock_stateid(lsp, &data->res.stateid))
+ 			goto out_restart;
+ 		break;
+-	case -NFS4ERR_BAD_STATEID:
+ 	case -NFS4ERR_OLD_STATEID:
++		if (data->arg.new_lock_owner != 0 &&
++			nfs4_refresh_open_old_stateid(&data->arg.open_stateid,
++					lsp->ls_state))
++			goto out_restart;
++		if (nfs4_refresh_lock_old_stateid(&data->arg.lock_stateid, lsp))
++			goto out_restart;
++		fallthrough;
++	case -NFS4ERR_BAD_STATEID:
+ 	case -NFS4ERR_STALE_STATEID:
+ 	case -NFS4ERR_EXPIRED:
+ 		if (data->arg.new_lock_owner != 0) {
+diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
+index c5dc0cd6f7031..96714e105d7bf 100644
+--- a/fs/nfsd/nfs4state.c
++++ b/fs/nfsd/nfs4state.c
+@@ -1368,9 +1368,9 @@ static void revoke_delegation(struct nfs4_delegation *dp)
+ 	WARN_ON(!list_empty(&dp->dl_recall_lru));
+ 
+ 	if (clp->cl_minorversion) {
++		spin_lock(&clp->cl_lock);
+ 		dp->dl_stid.sc_type = NFS4_REVOKED_DELEG_STID;
+ 		refcount_inc(&dp->dl_stid.sc_count);
+-		spin_lock(&clp->cl_lock);
+ 		list_add(&dp->dl_recall_lru, &clp->cl_revoked);
+ 		spin_unlock(&clp->cl_lock);
+ 	}
+diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
+index 155b34c4683c2..4c11046800ab4 100644
+--- a/fs/nfsd/vfs.c
++++ b/fs/nfsd/vfs.c
+@@ -321,7 +321,9 @@ nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap)
+ 				iap->ia_mode &= ~S_ISGID;
+ 		} else {
+ 			/* set ATTR_KILL_* bits and let VFS handle it */
+-			iap->ia_valid |= (ATTR_KILL_SUID | ATTR_KILL_SGID);
++			iap->ia_valid |= ATTR_KILL_SUID;
++			iap->ia_valid |=
++				setattr_should_drop_sgid(&init_user_ns, inode);
+ 		}
+ 	}
+ }
+diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h
+index 05f2cc03d03d9..b235d6833e27d 100644
+--- a/include/drm/display/drm_dp.h
++++ b/include/drm/display/drm_dp.h
+@@ -1525,7 +1525,7 @@ enum drm_dp_phy {
+ 
+ #define DP_BRANCH_OUI_HEADER_SIZE	0xc
+ #define DP_RECEIVER_CAP_SIZE		0xf
+-#define DP_DSC_RECEIVER_CAP_SIZE        0xf
++#define DP_DSC_RECEIVER_CAP_SIZE        0x10 /* DSC Capabilities 0x60 through 0x6F */
+ #define EDP_PSR_RECEIVER_CAP_SIZE	2
+ #define EDP_DISPLAY_CTL_CAP_SIZE	3
+ #define DP_LTTPR_COMMON_CAP_SIZE	8
+diff --git a/include/drm/drm_aperture.h b/include/drm/drm_aperture.h
+index 7096703c39493..cbe33b49fd5dc 100644
+--- a/include/drm/drm_aperture.h
++++ b/include/drm/drm_aperture.h
+@@ -13,14 +13,13 @@ int devm_aperture_acquire_from_firmware(struct drm_device *dev, resource_size_t
+ 					resource_size_t size);
+ 
+ int drm_aperture_remove_conflicting_framebuffers(resource_size_t base, resource_size_t size,
+-						 bool primary, const struct drm_driver *req_driver);
++						 const struct drm_driver *req_driver);
+ 
+ int drm_aperture_remove_conflicting_pci_framebuffers(struct pci_dev *pdev,
+ 						     const struct drm_driver *req_driver);
+ 
+ /**
+  * drm_aperture_remove_framebuffers - remove all existing framebuffers
+- * @primary: also kick vga16fb if present
+  * @req_driver: requesting DRM driver
+  *
+  * This function removes all graphics device drivers. Use this function on systems
+@@ -30,9 +29,9 @@ int drm_aperture_remove_conflicting_pci_framebuffers(struct pci_dev *pdev,
+  * 0 on success, or a negative errno code otherwise
+  */
+ static inline int
+-drm_aperture_remove_framebuffers(bool primary, const struct drm_driver *req_driver)
++drm_aperture_remove_framebuffers(const struct drm_driver *req_driver)
+ {
+-	return drm_aperture_remove_conflicting_framebuffers(0, (resource_size_t)-1, primary,
++	return drm_aperture_remove_conflicting_framebuffers(0, (resource_size_t)-1,
+ 							    req_driver);
+ }
+ 
+diff --git a/include/linux/clk.h b/include/linux/clk.h
+index 1ef0133242374..06f1b292f8a00 100644
+--- a/include/linux/clk.h
++++ b/include/linux/clk.h
+@@ -183,6 +183,39 @@ int clk_get_scaled_duty_cycle(struct clk *clk, unsigned int scale);
+  */
+ bool clk_is_match(const struct clk *p, const struct clk *q);
+ 
++/**
++ * clk_rate_exclusive_get - get exclusivity over the rate control of a
++ *                          producer
++ * @clk: clock source
++ *
++ * This function allows drivers to get exclusive control over the rate of a
++ * provider. It prevents any other consumer to execute, even indirectly,
++ * opereation which could alter the rate of the provider or cause glitches
++ *
++ * If exlusivity is claimed more than once on clock, even by the same driver,
++ * the rate effectively gets locked as exclusivity can't be preempted.
++ *
++ * Must not be called from within atomic context.
++ *
++ * Returns success (0) or negative errno.
++ */
++int clk_rate_exclusive_get(struct clk *clk);
++
++/**
++ * clk_rate_exclusive_put - release exclusivity over the rate control of a
++ *                          producer
++ * @clk: clock source
++ *
++ * This function allows drivers to release the exclusivity it previously got
++ * from clk_rate_exclusive_get()
++ *
++ * The caller must balance the number of clk_rate_exclusive_get() and
++ * clk_rate_exclusive_put() calls.
++ *
++ * Must not be called from within atomic context.
++ */
++void clk_rate_exclusive_put(struct clk *clk);
++
+ #else
+ 
+ static inline int clk_notifier_register(struct clk *clk,
+@@ -236,6 +269,13 @@ static inline bool clk_is_match(const struct clk *p, const struct clk *q)
+ 	return p == q;
+ }
+ 
++static inline int clk_rate_exclusive_get(struct clk *clk)
++{
++	return 0;
++}
++
++static inline void clk_rate_exclusive_put(struct clk *clk) {}
++
+ #endif
+ 
+ #ifdef CONFIG_HAVE_CLK_PREPARE
+@@ -583,38 +623,6 @@ struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id);
+  */
+ struct clk *devm_get_clk_from_child(struct device *dev,
+ 				    struct device_node *np, const char *con_id);
+-/**
+- * clk_rate_exclusive_get - get exclusivity over the rate control of a
+- *                          producer
+- * @clk: clock source
+- *
+- * This function allows drivers to get exclusive control over the rate of a
+- * provider. It prevents any other consumer to execute, even indirectly,
+- * opereation which could alter the rate of the provider or cause glitches
+- *
+- * If exlusivity is claimed more than once on clock, even by the same driver,
+- * the rate effectively gets locked as exclusivity can't be preempted.
+- *
+- * Must not be called from within atomic context.
+- *
+- * Returns success (0) or negative errno.
+- */
+-int clk_rate_exclusive_get(struct clk *clk);
+-
+-/**
+- * clk_rate_exclusive_put - release exclusivity over the rate control of a
+- *                          producer
+- * @clk: clock source
+- *
+- * This function allows drivers to release the exclusivity it previously got
+- * from clk_rate_exclusive_get()
+- *
+- * The caller must balance the number of clk_rate_exclusive_get() and
+- * clk_rate_exclusive_put() calls.
+- *
+- * Must not be called from within atomic context.
+- */
+-void clk_rate_exclusive_put(struct clk *clk);
+ 
+ /**
+  * clk_enable - inform the system when the clock source should be running.
+@@ -974,14 +982,6 @@ static inline void clk_bulk_put_all(int num_clks, struct clk_bulk_data *clks) {}
+ 
+ static inline void devm_clk_put(struct device *dev, struct clk *clk) {}
+ 
+-
+-static inline int clk_rate_exclusive_get(struct clk *clk)
+-{
+-	return 0;
+-}
+-
+-static inline void clk_rate_exclusive_put(struct clk *clk) {}
+-
+ static inline int clk_enable(struct clk *clk)
+ {
+ 	return 0;
+diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
+index d58e0476ee8e3..0348dba5680ef 100644
+--- a/include/linux/cpuset.h
++++ b/include/linux/cpuset.h
+@@ -71,8 +71,10 @@ extern void cpuset_init_smp(void);
+ extern void cpuset_force_rebuild(void);
+ extern void cpuset_update_active_cpus(void);
+ extern void cpuset_wait_for_hotplug(void);
+-extern void cpuset_read_lock(void);
+-extern void cpuset_read_unlock(void);
++extern void inc_dl_tasks_cs(struct task_struct *task);
++extern void dec_dl_tasks_cs(struct task_struct *task);
++extern void cpuset_lock(void);
++extern void cpuset_unlock(void);
+ extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
+ extern bool cpuset_cpus_allowed_fallback(struct task_struct *p);
+ extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
+@@ -196,8 +198,10 @@ static inline void cpuset_update_active_cpus(void)
+ 
+ static inline void cpuset_wait_for_hotplug(void) { }
+ 
+-static inline void cpuset_read_lock(void) { }
+-static inline void cpuset_read_unlock(void) { }
++static inline void inc_dl_tasks_cs(struct task_struct *task) { }
++static inline void dec_dl_tasks_cs(struct task_struct *task) { }
++static inline void cpuset_lock(void) { }
++static inline void cpuset_unlock(void) { }
+ 
+ static inline void cpuset_cpus_allowed(struct task_struct *p,
+ 				       struct cpumask *mask)
+diff --git a/include/linux/fs.h b/include/linux/fs.h
+index a2b5592c68284..26ea1a0a59a10 100644
+--- a/include/linux/fs.h
++++ b/include/linux/fs.h
+@@ -3120,6 +3120,8 @@ extern struct inode *new_inode(struct super_block *sb);
+ extern void free_inode_nonrcu(struct inode *inode);
+ extern int setattr_should_drop_suidgid(struct user_namespace *, struct inode *);
+ extern int file_remove_privs(struct file *);
++int setattr_should_drop_sgid(struct user_namespace *mnt_userns,
++			     const struct inode *inode);
+ 
+ /*
+  * This must be used for allocating filesystems specific inodes to set
+diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h
+index 0b7242370b567..ebb1608d9dcd2 100644
+--- a/include/linux/jbd2.h
++++ b/include/linux/jbd2.h
+@@ -622,12 +622,6 @@ struct transaction_s
+ 	 */
+ 	struct journal_head	*t_checkpoint_list;
+ 
+-	/*
+-	 * Doubly-linked circular list of all buffers submitted for IO while
+-	 * checkpointing. [j_list_lock]
+-	 */
+-	struct journal_head	*t_checkpoint_io_list;
+-
+ 	/*
+ 	 * Doubly-linked circular list of metadata buffers being
+ 	 * shadowed by log IO.  The IO buffers on the iobuf list and
+@@ -1441,6 +1435,7 @@ extern void jbd2_journal_commit_transaction(journal_t *);
+ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, bool destroy);
+ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, unsigned long *nr_to_scan);
+ int __jbd2_journal_remove_checkpoint(struct journal_head *);
++int jbd2_journal_try_remove_checkpoint(struct journal_head *jh);
+ void jbd2_journal_destroy_checkpoint(journal_t *journal);
+ void __jbd2_journal_insert_checkpoint(struct journal_head *, transaction_t *);
+ 
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index b8ed44f401b58..104ec00823da8 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -1727,6 +1727,25 @@ static inline size_t folio_size(struct folio *folio)
+ 	return PAGE_SIZE << folio_order(folio);
+ }
+ 
++/**
++ * folio_estimated_sharers - Estimate the number of sharers of a folio.
++ * @folio: The folio.
++ *
++ * folio_estimated_sharers() aims to serve as a function to efficiently
++ * estimate the number of processes sharing a folio. This is done by
++ * looking at the precise mapcount of the first subpage in the folio, and
++ * assuming the other subpages are the same. This may not be true for large
++ * folios. If you want exact mapcounts for exact calculations, look at
++ * page_mapcount() or folio_total_mapcount().
++ *
++ * Return: The estimated number of processes sharing a folio.
++ */
++static inline int folio_estimated_sharers(struct folio *folio)
++{
++	return page_mapcount(folio_page(folio, 0));
++}
++
++
+ #ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE
+ static inline int arch_make_page_accessible(struct page *page)
+ {
+@@ -3091,6 +3110,16 @@ static inline bool gup_must_unshare(unsigned int flags, struct page *page)
+ 	if (IS_ENABLED(CONFIG_HAVE_FAST_GUP))
+ 		smp_rmb();
+ 
++	/*
++	 * During GUP-fast we might not get called on the head page for a
++	 * hugetlb page that is mapped using cont-PTE, because GUP-fast does
++	 * not work with the abstracted hugetlb PTEs that always point at the
++	 * head page. For hugetlb, PageAnonExclusive only applies on the head
++	 * page (as it cannot be partially COW-shared), so lookup the head page.
++	 */
++	if (unlikely(!PageHead(page) && PageHuge(page)))
++		page = compound_head(page);
++
+ 	/*
+ 	 * Note that PageKsm() pages cannot be exclusive, and consequently,
+ 	 * cannot get pinned.
+diff --git a/include/linux/raid_class.h b/include/linux/raid_class.h
+index 5cdfcb873a8f0..772d45b2a60a0 100644
+--- a/include/linux/raid_class.h
++++ b/include/linux/raid_class.h
+@@ -77,7 +77,3 @@ DEFINE_RAID_ATTRIBUTE(enum raid_state, state)
+ 	
+ struct raid_template *raid_class_attach(struct raid_function_template *);
+ void raid_class_release(struct raid_template *);
+-
+-int __must_check raid_component_add(struct raid_template *, struct device *,
+-				    struct device *);
+-
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index ffb6eb55cd135..0cac69902ec58 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -1846,7 +1846,9 @@ current_restore_flags(unsigned long orig_flags, unsigned long flags)
+ }
+ 
+ extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
+-extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_effective_cpus);
++extern int task_can_attach(struct task_struct *p);
++extern int dl_bw_alloc(int cpu, u64 dl_bw);
++extern void dl_bw_free(int cpu, u64 dl_bw);
+ #ifdef CONFIG_SMP
+ extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
+ extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
+diff --git a/include/net/bonding.h b/include/net/bonding.h
+index 17329a19f0c64..9a3ac960dfe15 100644
+--- a/include/net/bonding.h
++++ b/include/net/bonding.h
+@@ -727,23 +727,14 @@ static inline struct slave *bond_slave_has_mac(struct bonding *bond,
+ }
+ 
+ /* Caller must hold rcu_read_lock() for read */
+-static inline bool bond_slave_has_mac_rx(struct bonding *bond, const u8 *mac)
++static inline bool bond_slave_has_mac_rcu(struct bonding *bond, const u8 *mac)
+ {
+ 	struct list_head *iter;
+ 	struct slave *tmp;
+-	struct netdev_hw_addr *ha;
+ 
+ 	bond_for_each_slave_rcu(bond, tmp, iter)
+ 		if (ether_addr_equal_64bits(mac, tmp->dev->dev_addr))
+ 			return true;
+-
+-	if (netdev_uc_empty(bond->dev))
+-		return false;
+-
+-	netdev_for_each_uc_addr(ha, bond->dev)
+-		if (ether_addr_equal_64bits(mac, ha->addr))
+-			return true;
+-
+ 	return false;
+ }
+ 
+diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h
+index c8ef3b881f03d..c2432c2addc82 100644
+--- a/include/net/inet_sock.h
++++ b/include/net/inet_sock.h
+@@ -222,8 +222,8 @@ struct inet_sock {
+ 	__s16			uc_ttl;
+ 	__u16			cmsg_flags;
+ 	struct ip_options_rcu __rcu	*inet_opt;
++	atomic_t		inet_id;
+ 	__be16			inet_sport;
+-	__u16			inet_id;
+ 
+ 	__u8			tos;
+ 	__u8			min_ttl;
+diff --git a/include/net/ip.h b/include/net/ip.h
+index 530e7257e4389..1872f570abeda 100644
+--- a/include/net/ip.h
++++ b/include/net/ip.h
+@@ -532,8 +532,19 @@ static inline void ip_select_ident_segs(struct net *net, struct sk_buff *skb,
+ 	 * generator as much as we can.
+ 	 */
+ 	if (sk && inet_sk(sk)->inet_daddr) {
+-		iph->id = htons(inet_sk(sk)->inet_id);
+-		inet_sk(sk)->inet_id += segs;
++		int val;
++
++		/* avoid atomic operations for TCP,
++		 * as we hold socket lock at this point.
++		 */
++		if (sk_is_tcp(sk)) {
++			sock_owned_by_me(sk);
++			val = atomic_read(&inet_sk(sk)->inet_id);
++			atomic_set(&inet_sk(sk)->inet_id, val + segs);
++		} else {
++			val = atomic_add_return(segs, &inet_sk(sk)->inet_id);
++		}
++		iph->id = htons(val);
+ 		return;
+ 	}
+ 	if ((iph->frag_off & htons(IP_DF)) && !skb->ignore_df) {
+diff --git a/include/net/mac80211.h b/include/net/mac80211.h
+index 72b739dc6d530..8a338c33118f9 100644
+--- a/include/net/mac80211.h
++++ b/include/net/mac80211.h
+@@ -6444,6 +6444,7 @@ void ieee80211_stop_rx_ba_session(struct ieee80211_vif *vif, u16 ba_rx_bitmap,
+  * marks frames marked in the bitmap as having been filtered. Afterwards, it
+  * checks if any frames in the window starting from @ssn can now be released
+  * (in case they were only waiting for frames that were filtered.)
++ * (Only work correctly if @max_rx_aggregation_subframes <= 64 frames)
+  */
+ void ieee80211_mark_rx_ba_filtered_frames(struct ieee80211_sta *pubsta, u8 tid,
+ 					  u16 ssn, u64 filtered,
+diff --git a/include/net/rtnetlink.h b/include/net/rtnetlink.h
+index bf8bb33578250..9f881b74f32ed 100644
+--- a/include/net/rtnetlink.h
++++ b/include/net/rtnetlink.h
+@@ -189,8 +189,8 @@ struct net_device *rtnl_create_link(struct net *net, const char *ifname,
+ int rtnl_delete_link(struct net_device *dev);
+ int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm);
+ 
+-int rtnl_nla_parse_ifla(struct nlattr **tb, const struct nlattr *head, int len,
+-			struct netlink_ext_ack *exterr);
++int rtnl_nla_parse_ifinfomsg(struct nlattr **tb, const struct nlattr *nla_peer,
++			     struct netlink_ext_ack *exterr);
+ struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid);
+ 
+ #define MODULE_ALIAS_RTNL_LINK(kind) MODULE_ALIAS("rtnl-link-" kind)
+diff --git a/include/net/sock.h b/include/net/sock.h
+index 699408944952c..d1f936ed97556 100644
+--- a/include/net/sock.h
++++ b/include/net/sock.h
+@@ -1320,6 +1320,7 @@ struct proto {
+ 	/*
+ 	 * Pressure flag: try to collapse.
+ 	 * Technical note: it is used by multiple contexts non atomically.
++	 * Make sure to use READ_ONCE()/WRITE_ONCE() for all reads/writes.
+ 	 * All the __sk_mem_schedule() is of this nature: accounting
+ 	 * is strict, actions are advisory and have some latency.
+ 	 */
+@@ -1448,7 +1449,7 @@ static inline bool sk_has_memory_pressure(const struct sock *sk)
+ static inline bool sk_under_global_memory_pressure(const struct sock *sk)
+ {
+ 	return sk->sk_prot->memory_pressure &&
+-		!!*sk->sk_prot->memory_pressure;
++		!!READ_ONCE(*sk->sk_prot->memory_pressure);
+ }
+ 
+ static inline bool sk_under_memory_pressure(const struct sock *sk)
+@@ -1460,7 +1461,7 @@ static inline bool sk_under_memory_pressure(const struct sock *sk)
+ 	    mem_cgroup_under_socket_pressure(sk->sk_memcg))
+ 		return true;
+ 
+-	return !!*sk->sk_prot->memory_pressure;
++	return !!READ_ONCE(*sk->sk_prot->memory_pressure);
+ }
+ 
+ static inline long
+@@ -1537,7 +1538,7 @@ proto_memory_pressure(struct proto *prot)
+ {
+ 	if (!prot->memory_pressure)
+ 		return false;
+-	return !!*prot->memory_pressure;
++	return !!READ_ONCE(*prot->memory_pressure);
+ }
+ 
+ 
+diff --git a/include/trace/events/jbd2.h b/include/trace/events/jbd2.h
+index 8f5ee380d3093..5646ae15a957a 100644
+--- a/include/trace/events/jbd2.h
++++ b/include/trace/events/jbd2.h
+@@ -462,11 +462,9 @@ TRACE_EVENT(jbd2_shrink_scan_exit,
+ TRACE_EVENT(jbd2_shrink_checkpoint_list,
+ 
+ 	TP_PROTO(journal_t *journal, tid_t first_tid, tid_t tid, tid_t last_tid,
+-		 unsigned long nr_freed, unsigned long nr_scanned,
+-		 tid_t next_tid),
++		 unsigned long nr_freed, tid_t next_tid),
+ 
+-	TP_ARGS(journal, first_tid, tid, last_tid, nr_freed,
+-		nr_scanned, next_tid),
++	TP_ARGS(journal, first_tid, tid, last_tid, nr_freed, next_tid),
+ 
+ 	TP_STRUCT__entry(
+ 		__field(dev_t, dev)
+@@ -474,7 +472,6 @@ TRACE_EVENT(jbd2_shrink_checkpoint_list,
+ 		__field(tid_t, tid)
+ 		__field(tid_t, last_tid)
+ 		__field(unsigned long, nr_freed)
+-		__field(unsigned long, nr_scanned)
+ 		__field(tid_t, next_tid)
+ 	),
+ 
+@@ -484,15 +481,14 @@ TRACE_EVENT(jbd2_shrink_checkpoint_list,
+ 		__entry->tid		= tid;
+ 		__entry->last_tid	= last_tid;
+ 		__entry->nr_freed	= nr_freed;
+-		__entry->nr_scanned	= nr_scanned;
+ 		__entry->next_tid	= next_tid;
+ 	),
+ 
+ 	TP_printk("dev %d,%d shrink transaction %u-%u(%u) freed %lu "
+-		  "scanned %lu next transaction %u",
++		  "next transaction %u",
+ 		  MAJOR(__entry->dev), MINOR(__entry->dev),
+ 		  __entry->first_tid, __entry->tid, __entry->last_tid,
+-		  __entry->nr_freed, __entry->nr_scanned, __entry->next_tid)
++		  __entry->nr_freed, __entry->next_tid)
+ );
+ 
+ #endif /* _TRACE_JBD2_H */
+diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
+index 3526389ac2180..cd922d2bef5f5 100644
+--- a/io_uring/msg_ring.c
++++ b/io_uring/msg_ring.c
+@@ -15,6 +15,7 @@
+ 
+ struct io_msg {
+ 	struct file			*file;
++	struct file			*src_file;
+ 	u64 user_data;
+ 	u32 len;
+ 	u32 cmd;
+@@ -23,33 +24,12 @@ struct io_msg {
+ 	u32 flags;
+ };
+ 
+-static int io_msg_ring_data(struct io_kiocb *req)
++static void io_double_unlock_ctx(struct io_ring_ctx *octx)
+ {
+-	struct io_ring_ctx *target_ctx = req->file->private_data;
+-	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
+-
+-	if (msg->src_fd || msg->dst_fd || msg->flags)
+-		return -EINVAL;
+-	if (target_ctx->flags & IORING_SETUP_R_DISABLED)
+-		return -EBADFD;
+-
+-	if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0, true))
+-		return 0;
+-
+-	return -EOVERFLOW;
+-}
+-
+-static void io_double_unlock_ctx(struct io_ring_ctx *ctx,
+-				 struct io_ring_ctx *octx,
+-				 unsigned int issue_flags)
+-{
+-	if (issue_flags & IO_URING_F_UNLOCKED)
+-		mutex_unlock(&ctx->uring_lock);
+ 	mutex_unlock(&octx->uring_lock);
+ }
+ 
+-static int io_double_lock_ctx(struct io_ring_ctx *ctx,
+-			      struct io_ring_ctx *octx,
++static int io_double_lock_ctx(struct io_ring_ctx *octx,
+ 			      unsigned int issue_flags)
+ {
+ 	/*
+@@ -62,60 +42,86 @@ static int io_double_lock_ctx(struct io_ring_ctx *ctx,
+ 			return -EAGAIN;
+ 		return 0;
+ 	}
++	mutex_lock(&octx->uring_lock);
++	return 0;
++}
+ 
+-	/* Always grab smallest value ctx first. We know ctx != octx. */
+-	if (ctx < octx) {
+-		mutex_lock(&ctx->uring_lock);
+-		mutex_lock(&octx->uring_lock);
+-	} else {
+-		mutex_lock(&octx->uring_lock);
+-		mutex_lock(&ctx->uring_lock);
+-	}
++void io_msg_ring_cleanup(struct io_kiocb *req)
++{
++	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
+ 
+-	return 0;
++	if (WARN_ON_ONCE(!msg->src_file))
++		return;
++
++	fput(msg->src_file);
++	msg->src_file = NULL;
+ }
+ 
+-static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
++static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags)
+ {
+ 	struct io_ring_ctx *target_ctx = req->file->private_data;
+ 	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
+-	struct io_ring_ctx *ctx = req->ctx;
+-	unsigned long file_ptr;
+-	struct file *src_file;
+ 	int ret;
+ 
+-	if (msg->len)
+-		return -EINVAL;
+-	if (target_ctx == ctx)
++	if (msg->src_fd || msg->dst_fd || msg->flags)
+ 		return -EINVAL;
+ 	if (target_ctx->flags & IORING_SETUP_R_DISABLED)
+ 		return -EBADFD;
+ 
+-	ret = io_double_lock_ctx(ctx, target_ctx, issue_flags);
+-	if (unlikely(ret))
+-		return ret;
++	ret = -EOVERFLOW;
++	if (target_ctx->flags & IORING_SETUP_IOPOLL) {
++		if (unlikely(io_double_lock_ctx(target_ctx, issue_flags)))
++			return -EAGAIN;
++		if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0, true))
++			ret = 0;
++		io_double_unlock_ctx(target_ctx);
++	} else {
++		if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0, true))
++			ret = 0;
++	}
+ 
+-	ret = -EBADF;
+-	if (unlikely(msg->src_fd >= ctx->nr_user_files))
+-		goto out_unlock;
++	return ret;
++}
+ 
+-	msg->src_fd = array_index_nospec(msg->src_fd, ctx->nr_user_files);
+-	file_ptr = io_fixed_file_slot(&ctx->file_table, msg->src_fd)->file_ptr;
+-	if (!file_ptr)
+-		goto out_unlock;
++static struct file *io_msg_grab_file(struct io_kiocb *req, unsigned int issue_flags)
++{
++	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
++	struct io_ring_ctx *ctx = req->ctx;
++	struct file *file = NULL;
++	unsigned long file_ptr;
++	int idx = msg->src_fd;
++
++	io_ring_submit_lock(ctx, issue_flags);
++	if (likely(idx < ctx->nr_user_files)) {
++		idx = array_index_nospec(idx, ctx->nr_user_files);
++		file_ptr = io_fixed_file_slot(&ctx->file_table, idx)->file_ptr;
++		file = (struct file *) (file_ptr & FFS_MASK);
++		if (file)
++			get_file(file);
++	}
++	io_ring_submit_unlock(ctx, issue_flags);
++	return file;
++}
+ 
+-	src_file = (struct file *) (file_ptr & FFS_MASK);
+-	get_file(src_file);
++static int io_msg_install_complete(struct io_kiocb *req, unsigned int issue_flags)
++{
++	struct io_ring_ctx *target_ctx = req->file->private_data;
++	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
++	struct file *src_file = msg->src_file;
++	int ret;
++
++	if (unlikely(io_double_lock_ctx(target_ctx, issue_flags)))
++		return -EAGAIN;
+ 
+ 	ret = __io_fixed_fd_install(target_ctx, src_file, msg->dst_fd);
+-	if (ret < 0) {
+-		fput(src_file);
++	if (ret < 0)
+ 		goto out_unlock;
+-	}
++
++	msg->src_file = NULL;
++	req->flags &= ~REQ_F_NEED_CLEANUP;
+ 
+ 	if (msg->flags & IORING_MSG_RING_CQE_SKIP)
+ 		goto out_unlock;
+-
+ 	/*
+ 	 * If this fails, the target still received the file descriptor but
+ 	 * wasn't notified of the fact. This means that if this request
+@@ -125,10 +131,29 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
+ 	if (!io_post_aux_cqe(target_ctx, msg->user_data, ret, 0, true))
+ 		ret = -EOVERFLOW;
+ out_unlock:
+-	io_double_unlock_ctx(ctx, target_ctx, issue_flags);
++	io_double_unlock_ctx(target_ctx);
+ 	return ret;
+ }
+ 
++static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags)
++{
++	struct io_ring_ctx *target_ctx = req->file->private_data;
++	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
++	struct io_ring_ctx *ctx = req->ctx;
++	struct file *src_file = msg->src_file;
++
++	if (target_ctx == ctx)
++		return -EINVAL;
++	if (!src_file) {
++		src_file = io_msg_grab_file(req, issue_flags);
++		if (!src_file)
++			return -EBADF;
++		msg->src_file = src_file;
++		req->flags |= REQ_F_NEED_CLEANUP;
++	}
++	return io_msg_install_complete(req, issue_flags);
++}
++
+ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ {
+ 	struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg);
+@@ -136,6 +161,7 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+ 	if (unlikely(sqe->buf_index || sqe->personality))
+ 		return -EINVAL;
+ 
++	msg->src_file = NULL;
+ 	msg->user_data = READ_ONCE(sqe->off);
+ 	msg->len = READ_ONCE(sqe->len);
+ 	msg->cmd = READ_ONCE(sqe->addr);
+@@ -159,7 +185,7 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
+ 
+ 	switch (msg->cmd) {
+ 	case IORING_MSG_DATA:
+-		ret = io_msg_ring_data(req);
++		ret = io_msg_ring_data(req, issue_flags);
+ 		break;
+ 	case IORING_MSG_SEND_FD:
+ 		ret = io_msg_send_fd(req, issue_flags);
+diff --git a/io_uring/msg_ring.h b/io_uring/msg_ring.h
+index fb9601f202d07..3987ee6c0e5f1 100644
+--- a/io_uring/msg_ring.h
++++ b/io_uring/msg_ring.h
+@@ -2,3 +2,4 @@
+ 
+ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags);
++void io_msg_ring_cleanup(struct io_kiocb *req);
+diff --git a/io_uring/opdef.c b/io_uring/opdef.c
+index 04dd2c983fce4..3aa0d65c50e34 100644
+--- a/io_uring/opdef.c
++++ b/io_uring/opdef.c
+@@ -445,6 +445,7 @@ const struct io_op_def io_op_defs[] = {
+ 		.name			= "MSG_RING",
+ 		.prep			= io_msg_ring_prep,
+ 		.issue			= io_msg_ring,
++		.cleanup		= io_msg_ring_cleanup,
+ 	},
+ 	[IORING_OP_FSETXATTR] = {
+ 		.needs_file = 1,
+diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
+index 73f11e4db3a4d..97ecca43386d9 100644
+--- a/kernel/cgroup/cgroup.c
++++ b/kernel/cgroup/cgroup.c
+@@ -57,6 +57,7 @@
+ #include <linux/file.h>
+ #include <linux/fs_parser.h>
+ #include <linux/sched/cputime.h>
++#include <linux/sched/deadline.h>
+ #include <linux/psi.h>
+ #include <net/sock.h>
+ 
+@@ -6681,6 +6682,9 @@ void cgroup_exit(struct task_struct *tsk)
+ 	list_add_tail(&tsk->cg_list, &cset->dying_tasks);
+ 	cset->nr_tasks--;
+ 
++	if (dl_task(tsk))
++		dec_dl_tasks_cs(tsk);
++
+ 	WARN_ON_ONCE(cgroup_task_frozen(tsk));
+ 	if (unlikely(!(tsk->flags & PF_KTHREAD) &&
+ 		     test_bit(CGRP_FREEZE, &task_dfl_cgroup(tsk)->flags)))
+diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
+index e276db7228451..db3e05b6b4dd2 100644
+--- a/kernel/cgroup/cpuset.c
++++ b/kernel/cgroup/cpuset.c
+@@ -193,6 +193,14 @@ struct cpuset {
+ 	int use_parent_ecpus;
+ 	int child_ecpus_count;
+ 
++	/*
++	 * number of SCHED_DEADLINE tasks attached to this cpuset, so that we
++	 * know when to rebuild associated root domain bandwidth information.
++	 */
++	int nr_deadline_tasks;
++	int nr_migrate_dl_tasks;
++	u64 sum_migrate_dl_bw;
++
+ 	/* Invalid partition error code, not lock protected */
+ 	enum prs_errcode prs_err;
+ 
+@@ -245,6 +253,20 @@ static inline struct cpuset *parent_cs(struct cpuset *cs)
+ 	return css_cs(cs->css.parent);
+ }
+ 
++void inc_dl_tasks_cs(struct task_struct *p)
++{
++	struct cpuset *cs = task_cs(p);
++
++	cs->nr_deadline_tasks++;
++}
++
++void dec_dl_tasks_cs(struct task_struct *p)
++{
++	struct cpuset *cs = task_cs(p);
++
++	cs->nr_deadline_tasks--;
++}
++
+ /* bits in struct cpuset flags field */
+ typedef enum {
+ 	CS_ONLINE,
+@@ -366,22 +388,23 @@ static struct cpuset top_cpuset = {
+ 		if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
+ 
+ /*
+- * There are two global locks guarding cpuset structures - cpuset_rwsem and
++ * There are two global locks guarding cpuset structures - cpuset_mutex and
+  * callback_lock. We also require taking task_lock() when dereferencing a
+  * task's cpuset pointer. See "The task_lock() exception", at the end of this
+- * comment.  The cpuset code uses only cpuset_rwsem write lock.  Other
+- * kernel subsystems can use cpuset_read_lock()/cpuset_read_unlock() to
+- * prevent change to cpuset structures.
++ * comment.  The cpuset code uses only cpuset_mutex. Other kernel subsystems
++ * can use cpuset_lock()/cpuset_unlock() to prevent change to cpuset
++ * structures. Note that cpuset_mutex needs to be a mutex as it is used in
++ * paths that rely on priority inheritance (e.g. scheduler - on RT) for
++ * correctness.
+  *
+  * A task must hold both locks to modify cpusets.  If a task holds
+- * cpuset_rwsem, it blocks others wanting that rwsem, ensuring that it
+- * is the only task able to also acquire callback_lock and be able to
+- * modify cpusets.  It can perform various checks on the cpuset structure
+- * first, knowing nothing will change.  It can also allocate memory while
+- * just holding cpuset_rwsem.  While it is performing these checks, various
+- * callback routines can briefly acquire callback_lock to query cpusets.
+- * Once it is ready to make the changes, it takes callback_lock, blocking
+- * everyone else.
++ * cpuset_mutex, it blocks others, ensuring that it is the only task able to
++ * also acquire callback_lock and be able to modify cpusets.  It can perform
++ * various checks on the cpuset structure first, knowing nothing will change.
++ * It can also allocate memory while just holding cpuset_mutex.  While it is
++ * performing these checks, various callback routines can briefly acquire
++ * callback_lock to query cpusets.  Once it is ready to make the changes, it
++ * takes callback_lock, blocking everyone else.
+  *
+  * Calls to the kernel memory allocator can not be made while holding
+  * callback_lock, as that would risk double tripping on callback_lock
+@@ -403,16 +426,16 @@ static struct cpuset top_cpuset = {
+  * guidelines for accessing subsystem state in kernel/cgroup.c
+  */
+ 
+-DEFINE_STATIC_PERCPU_RWSEM(cpuset_rwsem);
++static DEFINE_MUTEX(cpuset_mutex);
+ 
+-void cpuset_read_lock(void)
++void cpuset_lock(void)
+ {
+-	percpu_down_read(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ }
+ 
+-void cpuset_read_unlock(void)
++void cpuset_unlock(void)
+ {
+-	percpu_up_read(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ static DEFINE_SPINLOCK(callback_lock);
+@@ -496,7 +519,7 @@ static inline bool partition_is_populated(struct cpuset *cs,
+  * One way or another, we guarantee to return some non-empty subset
+  * of cpu_online_mask.
+  *
+- * Call with callback_lock or cpuset_rwsem held.
++ * Call with callback_lock or cpuset_mutex held.
+  */
+ static void guarantee_online_cpus(struct task_struct *tsk,
+ 				  struct cpumask *pmask)
+@@ -538,7 +561,7 @@ out_unlock:
+  * One way or another, we guarantee to return some non-empty subset
+  * of node_states[N_MEMORY].
+  *
+- * Call with callback_lock or cpuset_rwsem held.
++ * Call with callback_lock or cpuset_mutex held.
+  */
+ static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
+ {
+@@ -550,7 +573,7 @@ static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
+ /*
+  * update task's spread flag if cpuset's page/slab spread flag is set
+  *
+- * Call with callback_lock or cpuset_rwsem held. The check can be skipped
++ * Call with callback_lock or cpuset_mutex held. The check can be skipped
+  * if on default hierarchy.
+  */
+ static void cpuset_update_task_spread_flags(struct cpuset *cs,
+@@ -575,7 +598,7 @@ static void cpuset_update_task_spread_flags(struct cpuset *cs,
+  *
+  * One cpuset is a subset of another if all its allowed CPUs and
+  * Memory Nodes are a subset of the other, and its exclusive flags
+- * are only set if the other's are set.  Call holding cpuset_rwsem.
++ * are only set if the other's are set.  Call holding cpuset_mutex.
+  */
+ 
+ static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
+@@ -713,7 +736,7 @@ out:
+  * If we replaced the flag and mask values of the current cpuset
+  * (cur) with those values in the trial cpuset (trial), would
+  * our various subset and exclusive rules still be valid?  Presumes
+- * cpuset_rwsem held.
++ * cpuset_mutex held.
+  *
+  * 'cur' is the address of an actual, in-use cpuset.  Operations
+  * such as list traversal that depend on the actual address of the
+@@ -829,7 +852,7 @@ static void update_domain_attr_tree(struct sched_domain_attr *dattr,
+ 	rcu_read_unlock();
+ }
+ 
+-/* Must be called with cpuset_rwsem held.  */
++/* Must be called with cpuset_mutex held.  */
+ static inline int nr_cpusets(void)
+ {
+ 	/* jump label reference count + the top-level cpuset */
+@@ -855,7 +878,7 @@ static inline int nr_cpusets(void)
+  * domains when operating in the severe memory shortage situations
+  * that could cause allocation failures below.
+  *
+- * Must be called with cpuset_rwsem held.
++ * Must be called with cpuset_mutex held.
+  *
+  * The three key local variables below are:
+  *    cp - cpuset pointer, used (together with pos_css) to perform a
+@@ -1066,11 +1089,14 @@ done:
+ 	return ndoms;
+ }
+ 
+-static void update_tasks_root_domain(struct cpuset *cs)
++static void dl_update_tasks_root_domain(struct cpuset *cs)
+ {
+ 	struct css_task_iter it;
+ 	struct task_struct *task;
+ 
++	if (cs->nr_deadline_tasks == 0)
++		return;
++
+ 	css_task_iter_start(&cs->css, 0, &it);
+ 
+ 	while ((task = css_task_iter_next(&it)))
+@@ -1079,12 +1105,12 @@ static void update_tasks_root_domain(struct cpuset *cs)
+ 	css_task_iter_end(&it);
+ }
+ 
+-static void rebuild_root_domains(void)
++static void dl_rebuild_rd_accounting(void)
+ {
+ 	struct cpuset *cs = NULL;
+ 	struct cgroup_subsys_state *pos_css;
+ 
+-	percpu_rwsem_assert_held(&cpuset_rwsem);
++	lockdep_assert_held(&cpuset_mutex);
+ 	lockdep_assert_cpus_held();
+ 	lockdep_assert_held(&sched_domains_mutex);
+ 
+@@ -1107,7 +1133,7 @@ static void rebuild_root_domains(void)
+ 
+ 		rcu_read_unlock();
+ 
+-		update_tasks_root_domain(cs);
++		dl_update_tasks_root_domain(cs);
+ 
+ 		rcu_read_lock();
+ 		css_put(&cs->css);
+@@ -1121,7 +1147,7 @@ partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
+ {
+ 	mutex_lock(&sched_domains_mutex);
+ 	partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
+-	rebuild_root_domains();
++	dl_rebuild_rd_accounting();
+ 	mutex_unlock(&sched_domains_mutex);
+ }
+ 
+@@ -1134,7 +1160,7 @@ partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
+  * 'cpus' is removed, then call this routine to rebuild the
+  * scheduler's dynamic sched domains.
+  *
+- * Call with cpuset_rwsem held.  Takes cpus_read_lock().
++ * Call with cpuset_mutex held.  Takes cpus_read_lock().
+  */
+ static void rebuild_sched_domains_locked(void)
+ {
+@@ -1145,7 +1171,7 @@ static void rebuild_sched_domains_locked(void)
+ 	int ndoms;
+ 
+ 	lockdep_assert_cpus_held();
+-	percpu_rwsem_assert_held(&cpuset_rwsem);
++	lockdep_assert_held(&cpuset_mutex);
+ 
+ 	/*
+ 	 * If we have raced with CPU hotplug, return early to avoid
+@@ -1196,9 +1222,9 @@ static void rebuild_sched_domains_locked(void)
+ void rebuild_sched_domains(void)
+ {
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	rebuild_sched_domains_locked();
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ }
+ 
+@@ -1208,7 +1234,7 @@ void rebuild_sched_domains(void)
+  * @new_cpus: the temp variable for the new effective_cpus mask
+  *
+  * Iterate through each task of @cs updating its cpus_allowed to the
+- * effective cpuset's.  As this function is called with cpuset_rwsem held,
++ * effective cpuset's.  As this function is called with cpuset_mutex held,
+  * cpuset membership stays stable.
+  */
+ static void update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
+@@ -1317,7 +1343,7 @@ static int update_parent_subparts_cpumask(struct cpuset *cs, int cmd,
+ 	int old_prs, new_prs;
+ 	int part_error = PERR_NONE;	/* Partition error? */
+ 
+-	percpu_rwsem_assert_held(&cpuset_rwsem);
++	lockdep_assert_held(&cpuset_mutex);
+ 
+ 	/*
+ 	 * The parent must be a partition root.
+@@ -1540,7 +1566,7 @@ static int update_parent_subparts_cpumask(struct cpuset *cs, int cmd,
+  *
+  * On legacy hierarchy, effective_cpus will be the same with cpu_allowed.
+  *
+- * Called with cpuset_rwsem held
++ * Called with cpuset_mutex held
+  */
+ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
+ 				 bool force)
+@@ -1700,7 +1726,7 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
+ 	struct cpuset *sibling;
+ 	struct cgroup_subsys_state *pos_css;
+ 
+-	percpu_rwsem_assert_held(&cpuset_rwsem);
++	lockdep_assert_held(&cpuset_mutex);
+ 
+ 	/*
+ 	 * Check all its siblings and call update_cpumasks_hier()
+@@ -1950,12 +1976,12 @@ static void *cpuset_being_rebound;
+  * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
+  *
+  * Iterate through each task of @cs updating its mems_allowed to the
+- * effective cpuset's.  As this function is called with cpuset_rwsem held,
++ * effective cpuset's.  As this function is called with cpuset_mutex held,
+  * cpuset membership stays stable.
+  */
+ static void update_tasks_nodemask(struct cpuset *cs)
+ {
+-	static nodemask_t newmems;	/* protected by cpuset_rwsem */
++	static nodemask_t newmems;	/* protected by cpuset_mutex */
+ 	struct css_task_iter it;
+ 	struct task_struct *task;
+ 
+@@ -1968,7 +1994,7 @@ static void update_tasks_nodemask(struct cpuset *cs)
+ 	 * take while holding tasklist_lock.  Forks can happen - the
+ 	 * mpol_dup() cpuset_being_rebound check will catch such forks,
+ 	 * and rebind their vma mempolicies too.  Because we still hold
+-	 * the global cpuset_rwsem, we know that no other rebind effort
++	 * the global cpuset_mutex, we know that no other rebind effort
+ 	 * will be contending for the global variable cpuset_being_rebound.
+ 	 * It's ok if we rebind the same mm twice; mpol_rebind_mm()
+ 	 * is idempotent.  Also migrate pages in each mm to new nodes.
+@@ -2014,7 +2040,7 @@ static void update_tasks_nodemask(struct cpuset *cs)
+  *
+  * On legacy hierarchy, effective_mems will be the same with mems_allowed.
+  *
+- * Called with cpuset_rwsem held
++ * Called with cpuset_mutex held
+  */
+ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
+ {
+@@ -2067,7 +2093,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
+  * mempolicies and if the cpuset is marked 'memory_migrate',
+  * migrate the tasks pages to the new memory.
+  *
+- * Call with cpuset_rwsem held. May take callback_lock during call.
++ * Call with cpuset_mutex held. May take callback_lock during call.
+  * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
+  * lock each such tasks mm->mmap_lock, scan its vma's and rebind
+  * their mempolicies to the cpusets new mems_allowed.
+@@ -2159,7 +2185,7 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
+  * @cs: the cpuset in which each task's spread flags needs to be changed
+  *
+  * Iterate through each task of @cs updating its spread flags.  As this
+- * function is called with cpuset_rwsem held, cpuset membership stays
++ * function is called with cpuset_mutex held, cpuset membership stays
+  * stable.
+  */
+ static void update_tasks_flags(struct cpuset *cs)
+@@ -2179,7 +2205,7 @@ static void update_tasks_flags(struct cpuset *cs)
+  * cs:		the cpuset to update
+  * turning_on: 	whether the flag is being set or cleared
+  *
+- * Call with cpuset_rwsem held.
++ * Call with cpuset_mutex held.
+  */
+ 
+ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
+@@ -2229,7 +2255,7 @@ out:
+  * @new_prs: new partition root state
+  * Return: 0 if successful, != 0 if error
+  *
+- * Call with cpuset_rwsem held.
++ * Call with cpuset_mutex held.
+  */
+ static int update_prstate(struct cpuset *cs, int new_prs)
+ {
+@@ -2467,19 +2493,26 @@ static int cpuset_can_attach_check(struct cpuset *cs)
+ 	return 0;
+ }
+ 
+-/* Called by cgroups to determine if a cpuset is usable; cpuset_rwsem held */
++static void reset_migrate_dl_data(struct cpuset *cs)
++{
++	cs->nr_migrate_dl_tasks = 0;
++	cs->sum_migrate_dl_bw = 0;
++}
++
++/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
+ static int cpuset_can_attach(struct cgroup_taskset *tset)
+ {
+ 	struct cgroup_subsys_state *css;
+-	struct cpuset *cs;
++	struct cpuset *cs, *oldcs;
+ 	struct task_struct *task;
+ 	int ret;
+ 
+ 	/* used later by cpuset_attach() */
+ 	cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
++	oldcs = cpuset_attach_old_cs;
+ 	cs = css_cs(css);
+ 
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	/* Check to see if task is allowed in the cpuset */
+ 	ret = cpuset_can_attach_check(cs);
+@@ -2487,21 +2520,46 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
+ 		goto out_unlock;
+ 
+ 	cgroup_taskset_for_each(task, css, tset) {
+-		ret = task_can_attach(task, cs->effective_cpus);
++		ret = task_can_attach(task);
+ 		if (ret)
+ 			goto out_unlock;
+ 		ret = security_task_setscheduler(task);
+ 		if (ret)
+ 			goto out_unlock;
++
++		if (dl_task(task)) {
++			cs->nr_migrate_dl_tasks++;
++			cs->sum_migrate_dl_bw += task->dl.dl_bw;
++		}
+ 	}
+ 
++	if (!cs->nr_migrate_dl_tasks)
++		goto out_success;
++
++	if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) {
++		int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
++
++		if (unlikely(cpu >= nr_cpu_ids)) {
++			reset_migrate_dl_data(cs);
++			ret = -EINVAL;
++			goto out_unlock;
++		}
++
++		ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
++		if (ret) {
++			reset_migrate_dl_data(cs);
++			goto out_unlock;
++		}
++	}
++
++out_success:
+ 	/*
+ 	 * Mark attach is in progress.  This makes validate_change() fail
+ 	 * changes which zero cpus/mems_allowed.
+ 	 */
+ 	cs->attach_in_progress++;
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	return ret;
+ }
+ 
+@@ -2513,15 +2571,23 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset)
+ 	cgroup_taskset_first(tset, &css);
+ 	cs = css_cs(css);
+ 
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	cs->attach_in_progress--;
+ 	if (!cs->attach_in_progress)
+ 		wake_up(&cpuset_attach_wq);
+-	percpu_up_write(&cpuset_rwsem);
++
++	if (cs->nr_migrate_dl_tasks) {
++		int cpu = cpumask_any(cs->effective_cpus);
++
++		dl_bw_free(cpu, cs->sum_migrate_dl_bw);
++		reset_migrate_dl_data(cs);
++	}
++
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ /*
+- * Protected by cpuset_rwsem. cpus_attach is used only by cpuset_attach_task()
++ * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach_task()
+  * but we can't allocate it dynamically there.  Define it global and
+  * allocate from cpuset_init().
+  */
+@@ -2530,7 +2596,7 @@ static nodemask_t cpuset_attach_nodemask_to;
+ 
+ static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
+ {
+-	percpu_rwsem_assert_held(&cpuset_rwsem);
++	lockdep_assert_held(&cpuset_mutex);
+ 
+ 	if (cs != &top_cpuset)
+ 		guarantee_online_cpus(task, cpus_attach);
+@@ -2558,7 +2624,7 @@ static void cpuset_attach(struct cgroup_taskset *tset)
+ 	cs = css_cs(css);
+ 
+ 	lockdep_assert_cpus_held();	/* see cgroup_attach_lock() */
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
+ 
+@@ -2594,11 +2660,17 @@ static void cpuset_attach(struct cgroup_taskset *tset)
+ 
+ 	cs->old_mems_allowed = cpuset_attach_nodemask_to;
+ 
++	if (cs->nr_migrate_dl_tasks) {
++		cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks;
++		oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks;
++		reset_migrate_dl_data(cs);
++	}
++
+ 	cs->attach_in_progress--;
+ 	if (!cs->attach_in_progress)
+ 		wake_up(&cpuset_attach_wq);
+ 
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ /* The various types of files and directories in a cpuset file system */
+@@ -2630,7 +2702,7 @@ static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+ 	int retval = 0;
+ 
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	if (!is_cpuset_online(cs)) {
+ 		retval = -ENODEV;
+ 		goto out_unlock;
+@@ -2666,7 +2738,7 @@ static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
+ 		break;
+ 	}
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ 	return retval;
+ }
+@@ -2679,7 +2751,7 @@ static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
+ 	int retval = -ENODEV;
+ 
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	if (!is_cpuset_online(cs))
+ 		goto out_unlock;
+ 
+@@ -2692,7 +2764,7 @@ static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
+ 		break;
+ 	}
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ 	return retval;
+ }
+@@ -2725,7 +2797,7 @@ static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
+ 	 * operation like this one can lead to a deadlock through kernfs
+ 	 * active_ref protection.  Let's break the protection.  Losing the
+ 	 * protection is okay as we check whether @cs is online after
+-	 * grabbing cpuset_rwsem anyway.  This only happens on the legacy
++	 * grabbing cpuset_mutex anyway.  This only happens on the legacy
+ 	 * hierarchies.
+ 	 */
+ 	css_get(&cs->css);
+@@ -2733,7 +2805,7 @@ static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
+ 	flush_work(&cpuset_hotplug_work);
+ 
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	if (!is_cpuset_online(cs))
+ 		goto out_unlock;
+ 
+@@ -2757,7 +2829,7 @@ static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
+ 
+ 	free_cpuset(trialcs);
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ 	kernfs_unbreak_active_protection(of->kn);
+ 	css_put(&cs->css);
+@@ -2905,13 +2977,13 @@ static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf,
+ 
+ 	css_get(&cs->css);
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	if (!is_cpuset_online(cs))
+ 		goto out_unlock;
+ 
+ 	retval = update_prstate(cs, val);
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ 	css_put(&cs->css);
+ 	return retval ?: nbytes;
+@@ -3124,7 +3196,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
+ 		return 0;
+ 
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	set_bit(CS_ONLINE, &cs->flags);
+ 	if (is_spread_page(parent))
+@@ -3175,7 +3247,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css)
+ 	cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
+ 	spin_unlock_irq(&callback_lock);
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ 	return 0;
+ }
+@@ -3196,7 +3268,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css)
+ 	struct cpuset *cs = css_cs(css);
+ 
+ 	cpus_read_lock();
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	if (is_partition_valid(cs))
+ 		update_prstate(cs, 0);
+@@ -3215,7 +3287,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css)
+ 	cpuset_dec();
+ 	clear_bit(CS_ONLINE, &cs->flags);
+ 
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	cpus_read_unlock();
+ }
+ 
+@@ -3228,7 +3300,7 @@ static void cpuset_css_free(struct cgroup_subsys_state *css)
+ 
+ static void cpuset_bind(struct cgroup_subsys_state *root_css)
+ {
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	spin_lock_irq(&callback_lock);
+ 
+ 	if (is_in_v2_mode()) {
+@@ -3241,7 +3313,7 @@ static void cpuset_bind(struct cgroup_subsys_state *root_css)
+ 	}
+ 
+ 	spin_unlock_irq(&callback_lock);
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ /*
+@@ -3262,14 +3334,14 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
+ 		return 0;
+ 
+ 	lockdep_assert_held(&cgroup_mutex);
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	/* Check to see if task is allowed in the cpuset */
+ 	ret = cpuset_can_attach_check(cs);
+ 	if (ret)
+ 		goto out_unlock;
+ 
+-	ret = task_can_attach(task, cs->effective_cpus);
++	ret = task_can_attach(task);
+ 	if (ret)
+ 		goto out_unlock;
+ 
+@@ -3283,7 +3355,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
+ 	 */
+ 	cs->attach_in_progress++;
+ out_unlock:
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 	return ret;
+ }
+ 
+@@ -3299,11 +3371,11 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
+ 	if (same_cs)
+ 		return;
+ 
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	cs->attach_in_progress--;
+ 	if (!cs->attach_in_progress)
+ 		wake_up(&cpuset_attach_wq);
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ /*
+@@ -3331,7 +3403,7 @@ static void cpuset_fork(struct task_struct *task)
+ 	}
+ 
+ 	/* CLONE_INTO_CGROUP */
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 	guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
+ 	cpuset_attach_task(cs, task);
+ 
+@@ -3339,7 +3411,7 @@ static void cpuset_fork(struct task_struct *task)
+ 	if (!cs->attach_in_progress)
+ 		wake_up(&cpuset_attach_wq);
+ 
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ struct cgroup_subsys cpuset_cgrp_subsys = {
+@@ -3369,8 +3441,6 @@ struct cgroup_subsys cpuset_cgrp_subsys = {
+ 
+ int __init cpuset_init(void)
+ {
+-	BUG_ON(percpu_init_rwsem(&cpuset_rwsem));
+-
+ 	BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
+ 	BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
+ 	BUG_ON(!zalloc_cpumask_var(&top_cpuset.subparts_cpus, GFP_KERNEL));
+@@ -3442,7 +3512,7 @@ hotplug_update_tasks_legacy(struct cpuset *cs,
+ 	is_empty = cpumask_empty(cs->cpus_allowed) ||
+ 		   nodes_empty(cs->mems_allowed);
+ 
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 
+ 	/*
+ 	 * Move tasks to the nearest ancestor with execution resources,
+@@ -3452,7 +3522,7 @@ hotplug_update_tasks_legacy(struct cpuset *cs,
+ 	if (is_empty)
+ 		remove_tasks_in_empty_cpuset(cs);
+ 
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ }
+ 
+ static void
+@@ -3503,14 +3573,14 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
+ retry:
+ 	wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
+ 
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	/*
+ 	 * We have raced with task attaching. We wait until attaching
+ 	 * is finished, so we won't attach a task to an empty cpuset.
+ 	 */
+ 	if (cs->attach_in_progress) {
+-		percpu_up_write(&cpuset_rwsem);
++		mutex_unlock(&cpuset_mutex);
+ 		goto retry;
+ 	}
+ 
+@@ -3604,7 +3674,7 @@ update_tasks:
+ 		hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems,
+ 					    cpus_updated, mems_updated);
+ 
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ }
+ 
+ /**
+@@ -3634,7 +3704,7 @@ static void cpuset_hotplug_workfn(struct work_struct *work)
+ 	if (on_dfl && !alloc_cpumasks(NULL, &tmp))
+ 		ptmp = &tmp;
+ 
+-	percpu_down_write(&cpuset_rwsem);
++	mutex_lock(&cpuset_mutex);
+ 
+ 	/* fetch the available cpus/mems and find out which changed how */
+ 	cpumask_copy(&new_cpus, cpu_active_mask);
+@@ -3691,7 +3761,7 @@ static void cpuset_hotplug_workfn(struct work_struct *work)
+ 		update_tasks_nodemask(&top_cpuset);
+ 	}
+ 
+-	percpu_up_write(&cpuset_rwsem);
++	mutex_unlock(&cpuset_mutex);
+ 
+ 	/* if cpus or mems changed, we need to propagate to descendants */
+ 	if (cpus_updated || mems_updated) {
+@@ -4101,7 +4171,7 @@ void __cpuset_memory_pressure_bump(void)
+  *  - Used for /proc/<pid>/cpuset.
+  *  - No need to task_lock(tsk) on this tsk->cpuset reference, as it
+  *    doesn't really matter if tsk->cpuset changes after we read it,
+- *    and we take cpuset_rwsem, keeping cpuset_attach() from changing it
++ *    and we take cpuset_mutex, keeping cpuset_attach() from changing it
+  *    anyway.
+  */
+ int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns,
+diff --git a/kernel/sched/core.c b/kernel/sched/core.c
+index b23dcbeacdf33..0f6a92737c912 100644
+--- a/kernel/sched/core.c
++++ b/kernel/sched/core.c
+@@ -7475,6 +7475,7 @@ static int __sched_setscheduler(struct task_struct *p,
+ 	int reset_on_fork;
+ 	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
+ 	struct rq *rq;
++	bool cpuset_locked = false;
+ 
+ 	/* The pi code expects interrupts enabled */
+ 	BUG_ON(pi && in_interrupt());
+@@ -7524,8 +7525,14 @@ recheck:
+ 			return retval;
+ 	}
+ 
+-	if (pi)
+-		cpuset_read_lock();
++	/*
++	 * SCHED_DEADLINE bandwidth accounting relies on stable cpusets
++	 * information.
++	 */
++	if (dl_policy(policy) || dl_policy(p->policy)) {
++		cpuset_locked = true;
++		cpuset_lock();
++	}
+ 
+ 	/*
+ 	 * Make sure no PI-waiters arrive (or leave) while we are
+@@ -7601,8 +7608,8 @@ change:
+ 	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
+ 		policy = oldpolicy = -1;
+ 		task_rq_unlock(rq, p, &rf);
+-		if (pi)
+-			cpuset_read_unlock();
++		if (cpuset_locked)
++			cpuset_unlock();
+ 		goto recheck;
+ 	}
+ 
+@@ -7669,7 +7676,8 @@ change:
+ 	task_rq_unlock(rq, p, &rf);
+ 
+ 	if (pi) {
+-		cpuset_read_unlock();
++		if (cpuset_locked)
++			cpuset_unlock();
+ 		rt_mutex_adjust_pi(p);
+ 	}
+ 
+@@ -7681,8 +7689,8 @@ change:
+ 
+ unlock:
+ 	task_rq_unlock(rq, p, &rf);
+-	if (pi)
+-		cpuset_read_unlock();
++	if (cpuset_locked)
++		cpuset_unlock();
+ 	return retval;
+ }
+ 
+@@ -9075,8 +9083,7 @@ int cpuset_cpumask_can_shrink(const struct cpumask *cur,
+ 	return ret;
+ }
+ 
+-int task_can_attach(struct task_struct *p,
+-		    const struct cpumask *cs_effective_cpus)
++int task_can_attach(struct task_struct *p)
+ {
+ 	int ret = 0;
+ 
+@@ -9089,21 +9096,9 @@ int task_can_attach(struct task_struct *p,
+ 	 * success of set_cpus_allowed_ptr() on all attached tasks
+ 	 * before cpus_mask may be changed.
+ 	 */
+-	if (p->flags & PF_NO_SETAFFINITY) {
++	if (p->flags & PF_NO_SETAFFINITY)
+ 		ret = -EINVAL;
+-		goto out;
+-	}
+ 
+-	if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
+-					      cs_effective_cpus)) {
+-		int cpu = cpumask_any_and(cpu_active_mask, cs_effective_cpus);
+-
+-		if (unlikely(cpu >= nr_cpu_ids))
+-			return -EINVAL;
+-		ret = dl_cpu_busy(cpu, p);
+-	}
+-
+-out:
+ 	return ret;
+ }
+ 
+@@ -9385,7 +9380,7 @@ static void cpuset_cpu_active(void)
+ static int cpuset_cpu_inactive(unsigned int cpu)
+ {
+ 	if (!cpuhp_tasks_frozen) {
+-		int ret = dl_cpu_busy(cpu, NULL);
++		int ret = dl_bw_check_overflow(cpu);
+ 
+ 		if (ret)
+ 			return ret;
+diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
+index f7d381b6c3133..9ce9810861ba5 100644
+--- a/kernel/sched/deadline.c
++++ b/kernel/sched/deadline.c
+@@ -16,6 +16,8 @@
+  *                    Fabio Checconi <fchecconi@gmail.com>
+  */
+ 
++#include <linux/cpuset.h>
++
+ /*
+  * Default limits for DL period; on the top end we guard against small util
+  * tasks still getting ridiculously long effective runtimes, on the bottom end we
+@@ -2597,6 +2599,12 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p)
+ 	if (task_on_rq_queued(p) && p->dl.dl_runtime)
+ 		task_non_contending(p);
+ 
++	/*
++	 * In case a task is setscheduled out from SCHED_DEADLINE we need to
++	 * keep track of that on its cpuset (for correct bandwidth tracking).
++	 */
++	dec_dl_tasks_cs(p);
++
+ 	if (!task_on_rq_queued(p)) {
+ 		/*
+ 		 * Inactive timer is armed. However, p is leaving DEADLINE and
+@@ -2637,6 +2645,12 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p)
+ 	if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1)
+ 		put_task_struct(p);
+ 
++	/*
++	 * In case a task is setscheduled to SCHED_DEADLINE we need to keep
++	 * track of that on its cpuset (for correct bandwidth tracking).
++	 */
++	inc_dl_tasks_cs(p);
++
+ 	/* If p is not queued we will update its parameters at next wakeup. */
+ 	if (!task_on_rq_queued(p)) {
+ 		add_rq_bw(&p->dl, &rq->dl);
+@@ -3023,26 +3037,38 @@ int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur,
+ 	return ret;
+ }
+ 
+-int dl_cpu_busy(int cpu, struct task_struct *p)
++enum dl_bw_request {
++	dl_bw_req_check_overflow = 0,
++	dl_bw_req_alloc,
++	dl_bw_req_free
++};
++
++static int dl_bw_manage(enum dl_bw_request req, int cpu, u64 dl_bw)
+ {
+-	unsigned long flags, cap;
++	unsigned long flags;
+ 	struct dl_bw *dl_b;
+-	bool overflow;
++	bool overflow = 0;
+ 
+ 	rcu_read_lock_sched();
+ 	dl_b = dl_bw_of(cpu);
+ 	raw_spin_lock_irqsave(&dl_b->lock, flags);
+-	cap = dl_bw_capacity(cpu);
+-	overflow = __dl_overflow(dl_b, cap, 0, p ? p->dl.dl_bw : 0);
+ 
+-	if (!overflow && p) {
+-		/*
+-		 * We reserve space for this task in the destination
+-		 * root_domain, as we can't fail after this point.
+-		 * We will free resources in the source root_domain
+-		 * later on (see set_cpus_allowed_dl()).
+-		 */
+-		__dl_add(dl_b, p->dl.dl_bw, dl_bw_cpus(cpu));
++	if (req == dl_bw_req_free) {
++		__dl_sub(dl_b, dl_bw, dl_bw_cpus(cpu));
++	} else {
++		unsigned long cap = dl_bw_capacity(cpu);
++
++		overflow = __dl_overflow(dl_b, cap, 0, dl_bw);
++
++		if (req == dl_bw_req_alloc && !overflow) {
++			/*
++			 * We reserve space in the destination
++			 * root_domain, as we can't fail after this point.
++			 * We will free resources in the source root_domain
++			 * later on (see set_cpus_allowed_dl()).
++			 */
++			__dl_add(dl_b, dl_bw, dl_bw_cpus(cpu));
++		}
+ 	}
+ 
+ 	raw_spin_unlock_irqrestore(&dl_b->lock, flags);
+@@ -3050,6 +3076,21 @@ int dl_cpu_busy(int cpu, struct task_struct *p)
+ 
+ 	return overflow ? -EBUSY : 0;
+ }
++
++int dl_bw_check_overflow(int cpu)
++{
++	return dl_bw_manage(dl_bw_req_check_overflow, cpu, 0);
++}
++
++int dl_bw_alloc(int cpu, u64 dl_bw)
++{
++	return dl_bw_manage(dl_bw_req_alloc, cpu, dl_bw);
++}
++
++void dl_bw_free(int cpu, u64 dl_bw)
++{
++	dl_bw_manage(dl_bw_req_free, cpu, dl_bw);
++}
+ #endif
+ 
+ #ifdef CONFIG_SCHED_DEBUG
+diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
+index d6d488e8eb554..b62d53d7c264f 100644
+--- a/kernel/sched/sched.h
++++ b/kernel/sched/sched.h
+@@ -330,7 +330,7 @@ extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr);
+ extern bool __checkparam_dl(const struct sched_attr *attr);
+ extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
+ extern int  dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
+-extern int  dl_cpu_busy(int cpu, struct task_struct *p);
++extern int  dl_bw_check_overflow(int cpu);
+ 
+ #ifdef CONFIG_CGROUP_SCHED
+ 
+diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
+index af33c5a4166d4..1a87cb70f1eb5 100644
+--- a/kernel/trace/trace.c
++++ b/kernel/trace/trace.c
+@@ -4128,8 +4128,15 @@ static void *s_start(struct seq_file *m, loff_t *pos)
+ 	 * will point to the same string as current_trace->name.
+ 	 */
+ 	mutex_lock(&trace_types_lock);
+-	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
++	if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name)) {
++		/* Close iter->trace before switching to the new current tracer */
++		if (iter->trace->close)
++			iter->trace->close(iter);
+ 		*iter->trace = *tr->current_trace;
++		/* Reopen the new current tracer */
++		if (iter->trace->open)
++			iter->trace->open(iter);
++	}
+ 	mutex_unlock(&trace_types_lock);
+ 
+ #ifdef CONFIG_TRACER_MAX_TRACE
+@@ -5189,11 +5196,17 @@ int tracing_set_cpumask(struct trace_array *tr,
+ 				!cpumask_test_cpu(cpu, tracing_cpumask_new)) {
+ 			atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
+ 			ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
++#ifdef CONFIG_TRACER_MAX_TRACE
++			ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu);
++#endif
+ 		}
+ 		if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
+ 				cpumask_test_cpu(cpu, tracing_cpumask_new)) {
+ 			atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
+ 			ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
++#ifdef CONFIG_TRACER_MAX_TRACE
++			ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu);
++#endif
+ 		}
+ 	}
+ 	arch_spin_unlock(&tr->max_lock);
+diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
+index 590b3d51afae9..ba37f768e2f27 100644
+--- a/kernel/trace/trace_irqsoff.c
++++ b/kernel/trace/trace_irqsoff.c
+@@ -231,7 +231,8 @@ static void irqsoff_trace_open(struct trace_iterator *iter)
+ {
+ 	if (is_graph(iter->tr))
+ 		graph_trace_open(iter);
+-
++	else
++		iter->private = NULL;
+ }
+ 
+ static void irqsoff_trace_close(struct trace_iterator *iter)
+diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
+index 330aee1c1a49e..0469a04a355f2 100644
+--- a/kernel/trace/trace_sched_wakeup.c
++++ b/kernel/trace/trace_sched_wakeup.c
+@@ -168,6 +168,8 @@ static void wakeup_trace_open(struct trace_iterator *iter)
+ {
+ 	if (is_graph(iter->tr))
+ 		graph_trace_open(iter);
++	else
++		iter->private = NULL;
+ }
+ 
+ static void wakeup_trace_close(struct trace_iterator *iter)
+diff --git a/lib/clz_ctz.c b/lib/clz_ctz.c
+index 0d3a686b5ba29..fb8c0c5c2bd27 100644
+--- a/lib/clz_ctz.c
++++ b/lib/clz_ctz.c
+@@ -28,36 +28,16 @@ int __weak __clzsi2(int val)
+ }
+ EXPORT_SYMBOL(__clzsi2);
+ 
+-int __weak __clzdi2(long val);
+-int __weak __ctzdi2(long val);
+-#if BITS_PER_LONG == 32
+-
+-int __weak __clzdi2(long val)
++int __weak __clzdi2(u64 val);
++int __weak __clzdi2(u64 val)
+ {
+-	return 32 - fls((int)val);
++	return 64 - fls64(val);
+ }
+ EXPORT_SYMBOL(__clzdi2);
+ 
+-int __weak __ctzdi2(long val)
++int __weak __ctzdi2(u64 val);
++int __weak __ctzdi2(u64 val)
+ {
+-	return __ffs((u32)val);
++	return __ffs64(val);
+ }
+ EXPORT_SYMBOL(__ctzdi2);
+-
+-#elif BITS_PER_LONG == 64
+-
+-int __weak __clzdi2(long val)
+-{
+-	return 64 - fls64((u64)val);
+-}
+-EXPORT_SYMBOL(__clzdi2);
+-
+-int __weak __ctzdi2(long val)
+-{
+-	return __ffs64((u64)val);
+-}
+-EXPORT_SYMBOL(__ctzdi2);
+-
+-#else
+-#error BITS_PER_LONG not 32 or 64
+-#endif
+diff --git a/lib/maple_tree.c b/lib/maple_tree.c
+index 47d0c95b9a01e..250b4c67fac8f 100644
+--- a/lib/maple_tree.c
++++ b/lib/maple_tree.c
+@@ -4333,6 +4333,9 @@ static inline bool mas_wr_append(struct ma_wr_state *wr_mas)
+ 	struct ma_state *mas = wr_mas->mas;
+ 	unsigned char node_pivots = mt_pivots[wr_mas->type];
+ 
++	if (mt_in_rcu(mas->tree))
++		return false;
++
+ 	if ((mas->index != wr_mas->r_min) && (mas->last == wr_mas->r_max)) {
+ 		if (new_end < node_pivots)
+ 			wr_mas->pivots[new_end] = wr_mas->pivots[end];
+diff --git a/lib/radix-tree.c b/lib/radix-tree.c
+index 3c78e1e8b2ad6..2ec38f08e4f0e 100644
+--- a/lib/radix-tree.c
++++ b/lib/radix-tree.c
+@@ -1134,7 +1134,6 @@ static void set_iter_tags(struct radix_tree_iter *iter,
+ void __rcu **radix_tree_iter_resume(void __rcu **slot,
+ 					struct radix_tree_iter *iter)
+ {
+-	slot++;
+ 	iter->index = __radix_tree_iter_add(iter, 1);
+ 	iter->next_index = iter->index;
+ 	iter->tags = 0;
+diff --git a/mm/madvise.c b/mm/madvise.c
+index d03e149ffe6e8..5973399b2f9b7 100644
+--- a/mm/madvise.c
++++ b/mm/madvise.c
+@@ -654,8 +654,8 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
+ 		 * deactivate all pages.
+ 		 */
+ 		if (folio_test_large(folio)) {
+-			if (folio_mapcount(folio) != 1)
+-				goto out;
++			if (folio_estimated_sharers(folio) != 1)
++				break;
+ 			folio_get(folio);
+ 			if (!folio_trylock(folio)) {
+ 				folio_put(folio);
+diff --git a/mm/memory-failure.c b/mm/memory-failure.c
+index 4457f9423e2c1..99de0328d1bed 100644
+--- a/mm/memory-failure.c
++++ b/mm/memory-failure.c
+@@ -2591,10 +2591,13 @@ retry:
+ 	if (ret > 0) {
+ 		ret = soft_offline_in_use_page(page);
+ 	} else if (ret == 0) {
+-		if (!page_handle_poison(page, true, false) && try_again) {
+-			try_again = false;
+-			flags &= ~MF_COUNT_INCREASED;
+-			goto retry;
++		if (!page_handle_poison(page, true, false)) {
++			if (try_again) {
++				try_again = false;
++				flags &= ~MF_COUNT_INCREASED;
++				goto retry;
++			}
++			ret = -EBUSY;
+ 		}
+ 	}
+ 
+diff --git a/mm/shmem.c b/mm/shmem.c
+index aba041a3df739..10365ced5b1fc 100644
+--- a/mm/shmem.c
++++ b/mm/shmem.c
+@@ -800,14 +800,16 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
+ 	XA_STATE(xas, &mapping->i_pages, start);
+ 	struct page *page;
+ 	unsigned long swapped = 0;
++	unsigned long max = end - 1;
+ 
+ 	rcu_read_lock();
+-	xas_for_each(&xas, page, end - 1) {
++	xas_for_each(&xas, page, max) {
+ 		if (xas_retry(&xas, page))
+ 			continue;
+ 		if (xa_is_value(page))
+ 			swapped++;
+-
++		if (xas.xa_index == max)
++			break;
+ 		if (need_resched()) {
+ 			xas_pause(&xas);
+ 			cond_resched_rcu();
+diff --git a/mm/vmalloc.c b/mm/vmalloc.c
+index d5dc361dc104d..80bd104a4d42e 100644
+--- a/mm/vmalloc.c
++++ b/mm/vmalloc.c
+@@ -2909,6 +2909,10 @@ void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot)
+ 		free_vm_area(area);
+ 		return NULL;
+ 	}
++
++	flush_cache_vmap((unsigned long)area->addr,
++			 (unsigned long)area->addr + count * PAGE_SIZE);
++
+ 	return area->addr;
+ }
+ EXPORT_SYMBOL_GPL(vmap_pfn);
+diff --git a/net/Makefile b/net/Makefile
+index 6a62e5b273781..0914bea9c335f 100644
+--- a/net/Makefile
++++ b/net/Makefile
+@@ -23,6 +23,7 @@ obj-$(CONFIG_BPFILTER)		+= bpfilter/
+ obj-$(CONFIG_PACKET)		+= packet/
+ obj-$(CONFIG_NET_KEY)		+= key/
+ obj-$(CONFIG_BRIDGE)		+= bridge/
++obj-$(CONFIG_NET_DEVLINK)	+= devlink/
+ obj-$(CONFIG_NET_DSA)		+= dsa/
+ obj-$(CONFIG_ATALK)		+= appletalk/
+ obj-$(CONFIG_X25)		+= x25/
+diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c
+index f1741fbfb6178..98a624f32b946 100644
+--- a/net/batman-adv/bat_v_elp.c
++++ b/net/batman-adv/bat_v_elp.c
+@@ -506,7 +506,7 @@ int batadv_v_elp_packet_recv(struct sk_buff *skb,
+ 	struct batadv_priv *bat_priv = netdev_priv(if_incoming->soft_iface);
+ 	struct batadv_elp_packet *elp_packet;
+ 	struct batadv_hard_iface *primary_if;
+-	struct ethhdr *ethhdr = (struct ethhdr *)skb_mac_header(skb);
++	struct ethhdr *ethhdr;
+ 	bool res;
+ 	int ret = NET_RX_DROP;
+ 
+@@ -514,6 +514,7 @@ int batadv_v_elp_packet_recv(struct sk_buff *skb,
+ 	if (!res)
+ 		goto free_skb;
+ 
++	ethhdr = eth_hdr(skb);
+ 	if (batadv_is_my_mac(bat_priv, ethhdr->h_source))
+ 		goto free_skb;
+ 
+diff --git a/net/batman-adv/bat_v_ogm.c b/net/batman-adv/bat_v_ogm.c
+index 033639df96d85..9f4815f4c8e89 100644
+--- a/net/batman-adv/bat_v_ogm.c
++++ b/net/batman-adv/bat_v_ogm.c
+@@ -124,8 +124,10 @@ static void batadv_v_ogm_send_to_if(struct sk_buff *skb,
+ {
+ 	struct batadv_priv *bat_priv = netdev_priv(hard_iface->soft_iface);
+ 
+-	if (hard_iface->if_status != BATADV_IF_ACTIVE)
++	if (hard_iface->if_status != BATADV_IF_ACTIVE) {
++		kfree_skb(skb);
+ 		return;
++	}
+ 
+ 	batadv_inc_counter(bat_priv, BATADV_CNT_MGMT_TX);
+ 	batadv_add_counter(bat_priv, BATADV_CNT_MGMT_TX_BYTES,
+@@ -986,7 +988,7 @@ int batadv_v_ogm_packet_recv(struct sk_buff *skb,
+ {
+ 	struct batadv_priv *bat_priv = netdev_priv(if_incoming->soft_iface);
+ 	struct batadv_ogm2_packet *ogm_packet;
+-	struct ethhdr *ethhdr = eth_hdr(skb);
++	struct ethhdr *ethhdr;
+ 	int ogm_offset;
+ 	u8 *packet_pos;
+ 	int ret = NET_RX_DROP;
+@@ -1000,6 +1002,7 @@ int batadv_v_ogm_packet_recv(struct sk_buff *skb,
+ 	if (!batadv_check_management_packet(skb, if_incoming, BATADV_OGM2_HLEN))
+ 		goto free_skb;
+ 
++	ethhdr = eth_hdr(skb);
+ 	if (batadv_is_my_mac(bat_priv, ethhdr->h_source))
+ 		goto free_skb;
+ 
+diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c
+index 41c1ad33d009f..24c9c0c3f3166 100644
+--- a/net/batman-adv/hard-interface.c
++++ b/net/batman-adv/hard-interface.c
+@@ -630,7 +630,19 @@ out:
+  */
+ void batadv_update_min_mtu(struct net_device *soft_iface)
+ {
+-	soft_iface->mtu = batadv_hardif_min_mtu(soft_iface);
++	struct batadv_priv *bat_priv = netdev_priv(soft_iface);
++	int limit_mtu;
++	int mtu;
++
++	mtu = batadv_hardif_min_mtu(soft_iface);
++
++	if (bat_priv->mtu_set_by_user)
++		limit_mtu = bat_priv->mtu_set_by_user;
++	else
++		limit_mtu = ETH_DATA_LEN;
++
++	mtu = min(mtu, limit_mtu);
++	dev_set_mtu(soft_iface, mtu);
+ 
+ 	/* Check if the local translate table should be cleaned up to match a
+ 	 * new (and smaller) MTU.
+diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c
+index a5e4a4e976cf3..86e0664e0511b 100644
+--- a/net/batman-adv/netlink.c
++++ b/net/batman-adv/netlink.c
+@@ -495,7 +495,10 @@ static int batadv_netlink_set_mesh(struct sk_buff *skb, struct genl_info *info)
+ 		attr = info->attrs[BATADV_ATTR_FRAGMENTATION_ENABLED];
+ 
+ 		atomic_set(&bat_priv->fragmentation, !!nla_get_u8(attr));
++
++		rtnl_lock();
+ 		batadv_update_min_mtu(bat_priv->soft_iface);
++		rtnl_unlock();
+ 	}
+ 
+ 	if (info->attrs[BATADV_ATTR_GW_BANDWIDTH_DOWN]) {
+diff --git a/net/batman-adv/soft-interface.c b/net/batman-adv/soft-interface.c
+index 0f5c0679b55a2..38d411a52f331 100644
+--- a/net/batman-adv/soft-interface.c
++++ b/net/batman-adv/soft-interface.c
+@@ -154,11 +154,14 @@ static int batadv_interface_set_mac_addr(struct net_device *dev, void *p)
+ 
+ static int batadv_interface_change_mtu(struct net_device *dev, int new_mtu)
+ {
++	struct batadv_priv *bat_priv = netdev_priv(dev);
++
+ 	/* check ranges */
+ 	if (new_mtu < 68 || new_mtu > batadv_hardif_min_mtu(dev))
+ 		return -EINVAL;
+ 
+ 	dev->mtu = new_mtu;
++	bat_priv->mtu_set_by_user = new_mtu;
+ 
+ 	return 0;
+ }
+diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c
+index 01d30c1e412c7..5d8cee74772fe 100644
+--- a/net/batman-adv/translation-table.c
++++ b/net/batman-adv/translation-table.c
+@@ -774,7 +774,6 @@ check_roaming:
+ 		if (roamed_back) {
+ 			batadv_tt_global_free(bat_priv, tt_global,
+ 					      "Roaming canceled");
+-			tt_global = NULL;
+ 		} else {
+ 			/* The global entry has to be marked as ROAMING and
+ 			 * has to be kept for consistency purpose
+diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
+index 758cd797a063b..76791815b26ba 100644
+--- a/net/batman-adv/types.h
++++ b/net/batman-adv/types.h
+@@ -1546,6 +1546,12 @@ struct batadv_priv {
+ 	/** @soft_iface: net device which holds this struct as private data */
+ 	struct net_device *soft_iface;
+ 
++	/**
++	 * @mtu_set_by_user: MTU was set once by user
++	 * protected by rtnl_lock
++	 */
++	int mtu_set_by_user;
++
+ 	/**
+ 	 * @bat_counters: mesh internal traffic statistic counters (see
+ 	 *  batadv_counters)
+diff --git a/net/can/isotp.c b/net/can/isotp.c
+index b3c2a49b189cc..8c97f4061ffd7 100644
+--- a/net/can/isotp.c
++++ b/net/can/isotp.c
+@@ -175,12 +175,6 @@ static bool isotp_register_rxid(struct isotp_sock *so)
+ 	return (isotp_bc_flags(so) == 0);
+ }
+ 
+-static bool isotp_register_txecho(struct isotp_sock *so)
+-{
+-	/* all modes but SF_BROADCAST register for tx echo skbs */
+-	return (isotp_bc_flags(so) != CAN_ISOTP_SF_BROADCAST);
+-}
+-
+ static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer)
+ {
+ 	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
+@@ -1176,7 +1170,7 @@ static int isotp_release(struct socket *sock)
+ 	lock_sock(sk);
+ 
+ 	/* remove current filters & unregister */
+-	if (so->bound && isotp_register_txecho(so)) {
++	if (so->bound) {
+ 		if (so->ifindex) {
+ 			struct net_device *dev;
+ 
+@@ -1293,14 +1287,12 @@ static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ 		can_rx_register(net, dev, rx_id, SINGLE_MASK(rx_id),
+ 				isotp_rcv, sk, "isotp", sk);
+ 
+-	if (isotp_register_txecho(so)) {
+-		/* no consecutive frame echo skb in flight */
+-		so->cfecho = 0;
++	/* no consecutive frame echo skb in flight */
++	so->cfecho = 0;
+ 
+-		/* register for echo skb's */
+-		can_rx_register(net, dev, tx_id, SINGLE_MASK(tx_id),
+-				isotp_rcv_echo, sk, "isotpe", sk);
+-	}
++	/* register for echo skb's */
++	can_rx_register(net, dev, tx_id, SINGLE_MASK(tx_id),
++			isotp_rcv_echo, sk, "isotpe", sk);
+ 
+ 	dev_put(dev);
+ 
+@@ -1521,7 +1513,7 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg,
+ 	case NETDEV_UNREGISTER:
+ 		lock_sock(sk);
+ 		/* remove current filters & unregister */
+-		if (so->bound && isotp_register_txecho(so)) {
++		if (so->bound) {
+ 			if (isotp_register_rxid(so))
+ 				can_rx_unregister(dev_net(dev), dev, so->rxid,
+ 						  SINGLE_MASK(so->rxid),
+diff --git a/net/can/raw.c b/net/can/raw.c
+index 4abab2c3011a3..8c104339d538d 100644
+--- a/net/can/raw.c
++++ b/net/can/raw.c
+@@ -84,6 +84,8 @@ struct raw_sock {
+ 	struct sock sk;
+ 	int bound;
+ 	int ifindex;
++	struct net_device *dev;
++	netdevice_tracker dev_tracker;
+ 	struct list_head notifier;
+ 	int loopback;
+ 	int recv_own_msgs;
+@@ -277,21 +279,24 @@ static void raw_notify(struct raw_sock *ro, unsigned long msg,
+ 	if (!net_eq(dev_net(dev), sock_net(sk)))
+ 		return;
+ 
+-	if (ro->ifindex != dev->ifindex)
++	if (ro->dev != dev)
+ 		return;
+ 
+ 	switch (msg) {
+ 	case NETDEV_UNREGISTER:
+ 		lock_sock(sk);
+ 		/* remove current filters & unregister */
+-		if (ro->bound)
++		if (ro->bound) {
+ 			raw_disable_allfilters(dev_net(dev), dev, sk);
++			netdev_put(dev, &ro->dev_tracker);
++		}
+ 
+ 		if (ro->count > 1)
+ 			kfree(ro->filter);
+ 
+ 		ro->ifindex = 0;
+ 		ro->bound = 0;
++		ro->dev = NULL;
+ 		ro->count = 0;
+ 		release_sock(sk);
+ 
+@@ -337,6 +342,7 @@ static int raw_init(struct sock *sk)
+ 
+ 	ro->bound            = 0;
+ 	ro->ifindex          = 0;
++	ro->dev              = NULL;
+ 
+ 	/* set default filter to single entry dfilter */
+ 	ro->dfilter.can_id   = 0;
+@@ -383,18 +389,14 @@ static int raw_release(struct socket *sock)
+ 	list_del(&ro->notifier);
+ 	spin_unlock(&raw_notifier_lock);
+ 
++	rtnl_lock();
+ 	lock_sock(sk);
+ 
+ 	/* remove current filters & unregister */
+ 	if (ro->bound) {
+-		if (ro->ifindex) {
+-			struct net_device *dev;
+-
+-			dev = dev_get_by_index(sock_net(sk), ro->ifindex);
+-			if (dev) {
+-				raw_disable_allfilters(dev_net(dev), dev, sk);
+-				dev_put(dev);
+-			}
++		if (ro->dev) {
++			raw_disable_allfilters(dev_net(ro->dev), ro->dev, sk);
++			netdev_put(ro->dev, &ro->dev_tracker);
+ 		} else {
+ 			raw_disable_allfilters(sock_net(sk), NULL, sk);
+ 		}
+@@ -405,6 +407,7 @@ static int raw_release(struct socket *sock)
+ 
+ 	ro->ifindex = 0;
+ 	ro->bound = 0;
++	ro->dev = NULL;
+ 	ro->count = 0;
+ 	free_percpu(ro->uniq);
+ 
+@@ -412,6 +415,8 @@ static int raw_release(struct socket *sock)
+ 	sock->sk = NULL;
+ 
+ 	release_sock(sk);
++	rtnl_unlock();
++
+ 	sock_put(sk);
+ 
+ 	return 0;
+@@ -422,6 +427,7 @@ static int raw_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ 	struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
+ 	struct sock *sk = sock->sk;
+ 	struct raw_sock *ro = raw_sk(sk);
++	struct net_device *dev = NULL;
+ 	int ifindex;
+ 	int err = 0;
+ 	int notify_enetdown = 0;
+@@ -431,24 +437,23 @@ static int raw_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ 	if (addr->can_family != AF_CAN)
+ 		return -EINVAL;
+ 
++	rtnl_lock();
+ 	lock_sock(sk);
+ 
+ 	if (ro->bound && addr->can_ifindex == ro->ifindex)
+ 		goto out;
+ 
+ 	if (addr->can_ifindex) {
+-		struct net_device *dev;
+-
+ 		dev = dev_get_by_index(sock_net(sk), addr->can_ifindex);
+ 		if (!dev) {
+ 			err = -ENODEV;
+ 			goto out;
+ 		}
+ 		if (dev->type != ARPHRD_CAN) {
+-			dev_put(dev);
+ 			err = -ENODEV;
+-			goto out;
++			goto out_put_dev;
+ 		}
++
+ 		if (!(dev->flags & IFF_UP))
+ 			notify_enetdown = 1;
+ 
+@@ -456,7 +461,9 @@ static int raw_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ 
+ 		/* filters set by default/setsockopt */
+ 		err = raw_enable_allfilters(sock_net(sk), dev, sk);
+-		dev_put(dev);
++		if (err)
++			goto out_put_dev;
++
+ 	} else {
+ 		ifindex = 0;
+ 
+@@ -467,26 +474,30 @@ static int raw_bind(struct socket *sock, struct sockaddr *uaddr, int len)
+ 	if (!err) {
+ 		if (ro->bound) {
+ 			/* unregister old filters */
+-			if (ro->ifindex) {
+-				struct net_device *dev;
+-
+-				dev = dev_get_by_index(sock_net(sk),
+-						       ro->ifindex);
+-				if (dev) {
+-					raw_disable_allfilters(dev_net(dev),
+-							       dev, sk);
+-					dev_put(dev);
+-				}
++			if (ro->dev) {
++				raw_disable_allfilters(dev_net(ro->dev),
++						       ro->dev, sk);
++				/* drop reference to old ro->dev */
++				netdev_put(ro->dev, &ro->dev_tracker);
+ 			} else {
+ 				raw_disable_allfilters(sock_net(sk), NULL, sk);
+ 			}
+ 		}
+ 		ro->ifindex = ifindex;
+ 		ro->bound = 1;
++		/* bind() ok -> hold a reference for new ro->dev */
++		ro->dev = dev;
++		if (ro->dev)
++			netdev_hold(ro->dev, &ro->dev_tracker, GFP_KERNEL);
+ 	}
+ 
+- out:
++out_put_dev:
++	/* remove potential reference from dev_get_by_index() */
++	if (dev)
++		dev_put(dev);
++out:
+ 	release_sock(sk);
++	rtnl_unlock();
+ 
+ 	if (notify_enetdown) {
+ 		sk->sk_err = ENETDOWN;
+@@ -552,9 +563,9 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
+ 		rtnl_lock();
+ 		lock_sock(sk);
+ 
+-		if (ro->bound && ro->ifindex) {
+-			dev = dev_get_by_index(sock_net(sk), ro->ifindex);
+-			if (!dev) {
++		dev = ro->dev;
++		if (ro->bound && dev) {
++			if (dev->reg_state != NETREG_REGISTERED) {
+ 				if (count > 1)
+ 					kfree(filter);
+ 				err = -ENODEV;
+@@ -595,7 +606,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
+ 		ro->count  = count;
+ 
+  out_fil:
+-		dev_put(dev);
+ 		release_sock(sk);
+ 		rtnl_unlock();
+ 
+@@ -613,9 +623,9 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
+ 		rtnl_lock();
+ 		lock_sock(sk);
+ 
+-		if (ro->bound && ro->ifindex) {
+-			dev = dev_get_by_index(sock_net(sk), ro->ifindex);
+-			if (!dev) {
++		dev = ro->dev;
++		if (ro->bound && dev) {
++			if (dev->reg_state != NETREG_REGISTERED) {
+ 				err = -ENODEV;
+ 				goto out_err;
+ 			}
+@@ -639,7 +649,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
+ 		ro->err_mask = err_mask;
+ 
+  out_err:
+-		dev_put(dev);
+ 		release_sock(sk);
+ 		rtnl_unlock();
+ 
+diff --git a/net/core/Makefile b/net/core/Makefile
+index 5857cec87b839..10edd66a8a372 100644
+--- a/net/core/Makefile
++++ b/net/core/Makefile
+@@ -33,7 +33,6 @@ obj-$(CONFIG_LWTUNNEL) += lwtunnel.o
+ obj-$(CONFIG_LWTUNNEL_BPF) += lwt_bpf.o
+ obj-$(CONFIG_DST_CACHE) += dst_cache.o
+ obj-$(CONFIG_HWBM) += hwbm.o
+-obj-$(CONFIG_NET_DEVLINK) += devlink.o
+ obj-$(CONFIG_GRO_CELLS) += gro_cells.o
+ obj-$(CONFIG_FAILOVER) += failover.o
+ obj-$(CONFIG_NET_SOCK_MSG) += skmsg.o
+diff --git a/net/core/devlink.c b/net/core/devlink.c
+deleted file mode 100644
+index 5a4a4b34ac15c..0000000000000
+--- a/net/core/devlink.c
++++ /dev/null
+@@ -1,12547 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0-or-later
+-/*
+- * net/core/devlink.c - Network physical/parent device Netlink interface
+- *
+- * Heavily inspired by net/wireless/
+- * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+- * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
+- */
+-
+-#include <linux/etherdevice.h>
+-#include <linux/kernel.h>
+-#include <linux/module.h>
+-#include <linux/types.h>
+-#include <linux/slab.h>
+-#include <linux/gfp.h>
+-#include <linux/device.h>
+-#include <linux/list.h>
+-#include <linux/netdevice.h>
+-#include <linux/spinlock.h>
+-#include <linux/refcount.h>
+-#include <linux/workqueue.h>
+-#include <linux/u64_stats_sync.h>
+-#include <linux/timekeeping.h>
+-#include <rdma/ib_verbs.h>
+-#include <net/netlink.h>
+-#include <net/genetlink.h>
+-#include <net/rtnetlink.h>
+-#include <net/net_namespace.h>
+-#include <net/sock.h>
+-#include <net/devlink.h>
+-#define CREATE_TRACE_POINTS
+-#include <trace/events/devlink.h>
+-
+-#define DEVLINK_RELOAD_STATS_ARRAY_SIZE \
+-	(__DEVLINK_RELOAD_LIMIT_MAX * __DEVLINK_RELOAD_ACTION_MAX)
+-
+-struct devlink_dev_stats {
+-	u32 reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
+-	u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
+-};
+-
+-struct devlink {
+-	u32 index;
+-	struct list_head port_list;
+-	struct list_head rate_list;
+-	struct list_head sb_list;
+-	struct list_head dpipe_table_list;
+-	struct list_head resource_list;
+-	struct list_head param_list;
+-	struct list_head region_list;
+-	struct list_head reporter_list;
+-	struct mutex reporters_lock; /* protects reporter_list */
+-	struct devlink_dpipe_headers *dpipe_headers;
+-	struct list_head trap_list;
+-	struct list_head trap_group_list;
+-	struct list_head trap_policer_list;
+-	struct list_head linecard_list;
+-	struct mutex linecards_lock; /* protects linecard_list */
+-	const struct devlink_ops *ops;
+-	u64 features;
+-	struct xarray snapshot_ids;
+-	struct devlink_dev_stats stats;
+-	struct device *dev;
+-	possible_net_t _net;
+-	/* Serializes access to devlink instance specific objects such as
+-	 * port, sb, dpipe, resource, params, region, traps and more.
+-	 */
+-	struct mutex lock;
+-	struct lock_class_key lock_key;
+-	u8 reload_failed:1;
+-	refcount_t refcount;
+-	struct completion comp;
+-	struct rcu_head rcu;
+-	char priv[] __aligned(NETDEV_ALIGN);
+-};
+-
+-struct devlink_linecard_ops;
+-struct devlink_linecard_type;
+-
+-struct devlink_linecard {
+-	struct list_head list;
+-	struct devlink *devlink;
+-	unsigned int index;
+-	refcount_t refcount;
+-	const struct devlink_linecard_ops *ops;
+-	void *priv;
+-	enum devlink_linecard_state state;
+-	struct mutex state_lock; /* Protects state */
+-	const char *type;
+-	struct devlink_linecard_type *types;
+-	unsigned int types_count;
+-	struct devlink *nested_devlink;
+-};
+-
+-/**
+- * struct devlink_resource - devlink resource
+- * @name: name of the resource
+- * @id: id, per devlink instance
+- * @size: size of the resource
+- * @size_new: updated size of the resource, reload is needed
+- * @size_valid: valid in case the total size of the resource is valid
+- *              including its children
+- * @parent: parent resource
+- * @size_params: size parameters
+- * @list: parent list
+- * @resource_list: list of child resources
+- * @occ_get: occupancy getter callback
+- * @occ_get_priv: occupancy getter callback priv
+- */
+-struct devlink_resource {
+-	const char *name;
+-	u64 id;
+-	u64 size;
+-	u64 size_new;
+-	bool size_valid;
+-	struct devlink_resource *parent;
+-	struct devlink_resource_size_params size_params;
+-	struct list_head list;
+-	struct list_head resource_list;
+-	devlink_resource_occ_get_t *occ_get;
+-	void *occ_get_priv;
+-};
+-
+-void *devlink_priv(struct devlink *devlink)
+-{
+-	return &devlink->priv;
+-}
+-EXPORT_SYMBOL_GPL(devlink_priv);
+-
+-struct devlink *priv_to_devlink(void *priv)
+-{
+-	return container_of(priv, struct devlink, priv);
+-}
+-EXPORT_SYMBOL_GPL(priv_to_devlink);
+-
+-struct device *devlink_to_dev(const struct devlink *devlink)
+-{
+-	return devlink->dev;
+-}
+-EXPORT_SYMBOL_GPL(devlink_to_dev);
+-
+-static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
+-	{
+-		.name = "destination mac",
+-		.id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
+-		.bitwidth = 48,
+-	},
+-};
+-
+-struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
+-	.name = "ethernet",
+-	.id = DEVLINK_DPIPE_HEADER_ETHERNET,
+-	.fields = devlink_dpipe_fields_ethernet,
+-	.fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
+-	.global = true,
+-};
+-EXPORT_SYMBOL_GPL(devlink_dpipe_header_ethernet);
+-
+-static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
+-	{
+-		.name = "destination ip",
+-		.id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
+-		.bitwidth = 32,
+-	},
+-};
+-
+-struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
+-	.name = "ipv4",
+-	.id = DEVLINK_DPIPE_HEADER_IPV4,
+-	.fields = devlink_dpipe_fields_ipv4,
+-	.fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
+-	.global = true,
+-};
+-EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv4);
+-
+-static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
+-	{
+-		.name = "destination ip",
+-		.id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
+-		.bitwidth = 128,
+-	},
+-};
+-
+-struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
+-	.name = "ipv6",
+-	.id = DEVLINK_DPIPE_HEADER_IPV6,
+-	.fields = devlink_dpipe_fields_ipv6,
+-	.fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
+-	.global = true,
+-};
+-EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv6);
+-
+-EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
+-EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
+-EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
+-
+-static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
+-	[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
+-	[DEVLINK_PORT_FN_ATTR_STATE] =
+-		NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
+-				 DEVLINK_PORT_FN_STATE_ACTIVE),
+-};
+-
+-static const struct nla_policy devlink_selftest_nl_policy[DEVLINK_ATTR_SELFTEST_ID_MAX + 1] = {
+-	[DEVLINK_ATTR_SELFTEST_ID_FLASH] = { .type = NLA_FLAG },
+-};
+-
+-static DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
+-#define DEVLINK_REGISTERED XA_MARK_1
+-#define DEVLINK_UNREGISTERING XA_MARK_2
+-
+-/* devlink instances are open to the access from the user space after
+- * devlink_register() call. Such logical barrier allows us to have certain
+- * expectations related to locking.
+- *
+- * Before *_register() - we are in initialization stage and no parallel
+- * access possible to the devlink instance. All drivers perform that phase
+- * by implicitly holding device_lock.
+- *
+- * After *_register() - users and driver can access devlink instance at
+- * the same time.
+- */
+-#define ASSERT_DEVLINK_REGISTERED(d)                                           \
+-	WARN_ON_ONCE(!xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
+-#define ASSERT_DEVLINK_NOT_REGISTERED(d)                                       \
+-	WARN_ON_ONCE(xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
+-
+-struct net *devlink_net(const struct devlink *devlink)
+-{
+-	return read_pnet(&devlink->_net);
+-}
+-EXPORT_SYMBOL_GPL(devlink_net);
+-
+-static void __devlink_put_rcu(struct rcu_head *head)
+-{
+-	struct devlink *devlink = container_of(head, struct devlink, rcu);
+-
+-	complete(&devlink->comp);
+-}
+-
+-void devlink_put(struct devlink *devlink)
+-{
+-	if (refcount_dec_and_test(&devlink->refcount))
+-		/* Make sure unregister operation that may await the completion
+-		 * is unblocked only after all users are after the end of
+-		 * RCU grace period.
+-		 */
+-		call_rcu(&devlink->rcu, __devlink_put_rcu);
+-}
+-
+-struct devlink *__must_check devlink_try_get(struct devlink *devlink)
+-{
+-	if (refcount_inc_not_zero(&devlink->refcount))
+-		return devlink;
+-	return NULL;
+-}
+-
+-void devl_assert_locked(struct devlink *devlink)
+-{
+-	lockdep_assert_held(&devlink->lock);
+-}
+-EXPORT_SYMBOL_GPL(devl_assert_locked);
+-
+-#ifdef CONFIG_LOCKDEP
+-/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
+-bool devl_lock_is_held(struct devlink *devlink)
+-{
+-	return lockdep_is_held(&devlink->lock);
+-}
+-EXPORT_SYMBOL_GPL(devl_lock_is_held);
+-#endif
+-
+-void devl_lock(struct devlink *devlink)
+-{
+-	mutex_lock(&devlink->lock);
+-}
+-EXPORT_SYMBOL_GPL(devl_lock);
+-
+-int devl_trylock(struct devlink *devlink)
+-{
+-	return mutex_trylock(&devlink->lock);
+-}
+-EXPORT_SYMBOL_GPL(devl_trylock);
+-
+-void devl_unlock(struct devlink *devlink)
+-{
+-	mutex_unlock(&devlink->lock);
+-}
+-EXPORT_SYMBOL_GPL(devl_unlock);
+-
+-static struct devlink *
+-devlinks_xa_find_get(struct net *net, unsigned long *indexp, xa_mark_t filter,
+-		     void * (*xa_find_fn)(struct xarray *, unsigned long *,
+-					  unsigned long, xa_mark_t))
+-{
+-	struct devlink *devlink;
+-
+-	rcu_read_lock();
+-retry:
+-	devlink = xa_find_fn(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED);
+-	if (!devlink)
+-		goto unlock;
+-
+-	/* In case devlink_unregister() was already called and "unregistering"
+-	 * mark was set, do not allow to get a devlink reference here.
+-	 * This prevents live-lock of devlink_unregister() wait for completion.
+-	 */
+-	if (xa_get_mark(&devlinks, *indexp, DEVLINK_UNREGISTERING))
+-		goto retry;
+-
+-	/* For a possible retry, the xa_find_after() should be always used */
+-	xa_find_fn = xa_find_after;
+-	if (!devlink_try_get(devlink))
+-		goto retry;
+-	if (!net_eq(devlink_net(devlink), net)) {
+-		devlink_put(devlink);
+-		goto retry;
+-	}
+-unlock:
+-	rcu_read_unlock();
+-	return devlink;
+-}
+-
+-static struct devlink *devlinks_xa_find_get_first(struct net *net,
+-						  unsigned long *indexp,
+-						  xa_mark_t filter)
+-{
+-	return devlinks_xa_find_get(net, indexp, filter, xa_find);
+-}
+-
+-static struct devlink *devlinks_xa_find_get_next(struct net *net,
+-						 unsigned long *indexp,
+-						 xa_mark_t filter)
+-{
+-	return devlinks_xa_find_get(net, indexp, filter, xa_find_after);
+-}
+-
+-/* Iterate over devlink pointers which were possible to get reference to.
+- * devlink_put() needs to be called for each iterated devlink pointer
+- * in loop body in order to release the reference.
+- */
+-#define devlinks_xa_for_each_get(net, index, devlink, filter)			\
+-	for (index = 0,								\
+-	     devlink = devlinks_xa_find_get_first(net, &index, filter);		\
+-	     devlink; devlink = devlinks_xa_find_get_next(net, &index, filter))
+-
+-#define devlinks_xa_for_each_registered_get(net, index, devlink)		\
+-	devlinks_xa_for_each_get(net, index, devlink, DEVLINK_REGISTERED)
+-
+-static struct devlink *devlink_get_from_attrs(struct net *net,
+-					      struct nlattr **attrs)
+-{
+-	struct devlink *devlink;
+-	unsigned long index;
+-	char *busname;
+-	char *devname;
+-
+-	if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
+-		return ERR_PTR(-EINVAL);
+-
+-	busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
+-	devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
+-
+-	devlinks_xa_for_each_registered_get(net, index, devlink) {
+-		if (strcmp(devlink->dev->bus->name, busname) == 0 &&
+-		    strcmp(dev_name(devlink->dev), devname) == 0)
+-			return devlink;
+-		devlink_put(devlink);
+-	}
+-
+-	return ERR_PTR(-ENODEV);
+-}
+-
+-#define ASSERT_DEVLINK_PORT_REGISTERED(devlink_port)				\
+-	WARN_ON_ONCE(!(devlink_port)->registered)
+-#define ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port)			\
+-	WARN_ON_ONCE((devlink_port)->registered)
+-#define ASSERT_DEVLINK_PORT_INITIALIZED(devlink_port)				\
+-	WARN_ON_ONCE(!(devlink_port)->initialized)
+-
+-static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
+-						      unsigned int port_index)
+-{
+-	struct devlink_port *devlink_port;
+-
+-	list_for_each_entry(devlink_port, &devlink->port_list, list) {
+-		if (devlink_port->index == port_index)
+-			return devlink_port;
+-	}
+-	return NULL;
+-}
+-
+-static bool devlink_port_index_exists(struct devlink *devlink,
+-				      unsigned int port_index)
+-{
+-	return devlink_port_get_by_index(devlink, port_index);
+-}
+-
+-static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
+-							struct nlattr **attrs)
+-{
+-	if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
+-		u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
+-		struct devlink_port *devlink_port;
+-
+-		devlink_port = devlink_port_get_by_index(devlink, port_index);
+-		if (!devlink_port)
+-			return ERR_PTR(-ENODEV);
+-		return devlink_port;
+-	}
+-	return ERR_PTR(-EINVAL);
+-}
+-
+-static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
+-						       struct genl_info *info)
+-{
+-	return devlink_port_get_from_attrs(devlink, info->attrs);
+-}
+-
+-static inline bool
+-devlink_rate_is_leaf(struct devlink_rate *devlink_rate)
+-{
+-	return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF;
+-}
+-
+-static inline bool
+-devlink_rate_is_node(struct devlink_rate *devlink_rate)
+-{
+-	return devlink_rate->type == DEVLINK_RATE_TYPE_NODE;
+-}
+-
+-static struct devlink_rate *
+-devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
+-{
+-	struct devlink_rate *devlink_rate;
+-	struct devlink_port *devlink_port;
+-
+-	devlink_port = devlink_port_get_from_attrs(devlink, info->attrs);
+-	if (IS_ERR(devlink_port))
+-		return ERR_CAST(devlink_port);
+-	devlink_rate = devlink_port->devlink_rate;
+-	return devlink_rate ?: ERR_PTR(-ENODEV);
+-}
+-
+-static struct devlink_rate *
+-devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
+-{
+-	static struct devlink_rate *devlink_rate;
+-
+-	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
+-		if (devlink_rate_is_node(devlink_rate) &&
+-		    !strcmp(node_name, devlink_rate->name))
+-			return devlink_rate;
+-	}
+-	return ERR_PTR(-ENODEV);
+-}
+-
+-static struct devlink_rate *
+-devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
+-{
+-	const char *rate_node_name;
+-	size_t len;
+-
+-	if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME])
+-		return ERR_PTR(-EINVAL);
+-	rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]);
+-	len = strlen(rate_node_name);
+-	/* Name cannot be empty or decimal number */
+-	if (!len || strspn(rate_node_name, "0123456789") == len)
+-		return ERR_PTR(-EINVAL);
+-
+-	return devlink_rate_node_get_by_name(devlink, rate_node_name);
+-}
+-
+-static struct devlink_rate *
+-devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
+-{
+-	return devlink_rate_node_get_from_attrs(devlink, info->attrs);
+-}
+-
+-static struct devlink_rate *
+-devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
+-{
+-	struct nlattr **attrs = info->attrs;
+-
+-	if (attrs[DEVLINK_ATTR_PORT_INDEX])
+-		return devlink_rate_leaf_get_from_info(devlink, info);
+-	else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
+-		return devlink_rate_node_get_from_info(devlink, info);
+-	else
+-		return ERR_PTR(-EINVAL);
+-}
+-
+-static struct devlink_linecard *
+-devlink_linecard_get_by_index(struct devlink *devlink,
+-			      unsigned int linecard_index)
+-{
+-	struct devlink_linecard *devlink_linecard;
+-
+-	list_for_each_entry(devlink_linecard, &devlink->linecard_list, list) {
+-		if (devlink_linecard->index == linecard_index)
+-			return devlink_linecard;
+-	}
+-	return NULL;
+-}
+-
+-static bool devlink_linecard_index_exists(struct devlink *devlink,
+-					  unsigned int linecard_index)
+-{
+-	return devlink_linecard_get_by_index(devlink, linecard_index);
+-}
+-
+-static struct devlink_linecard *
+-devlink_linecard_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
+-{
+-	if (attrs[DEVLINK_ATTR_LINECARD_INDEX]) {
+-		u32 linecard_index = nla_get_u32(attrs[DEVLINK_ATTR_LINECARD_INDEX]);
+-		struct devlink_linecard *linecard;
+-
+-		mutex_lock(&devlink->linecards_lock);
+-		linecard = devlink_linecard_get_by_index(devlink, linecard_index);
+-		if (linecard)
+-			refcount_inc(&linecard->refcount);
+-		mutex_unlock(&devlink->linecards_lock);
+-		if (!linecard)
+-			return ERR_PTR(-ENODEV);
+-		return linecard;
+-	}
+-	return ERR_PTR(-EINVAL);
+-}
+-
+-static struct devlink_linecard *
+-devlink_linecard_get_from_info(struct devlink *devlink, struct genl_info *info)
+-{
+-	return devlink_linecard_get_from_attrs(devlink, info->attrs);
+-}
+-
+-static void devlink_linecard_put(struct devlink_linecard *linecard)
+-{
+-	if (refcount_dec_and_test(&linecard->refcount)) {
+-		mutex_destroy(&linecard->state_lock);
+-		kfree(linecard);
+-	}
+-}
+-
+-struct devlink_sb {
+-	struct list_head list;
+-	unsigned int index;
+-	u32 size;
+-	u16 ingress_pools_count;
+-	u16 egress_pools_count;
+-	u16 ingress_tc_count;
+-	u16 egress_tc_count;
+-};
+-
+-static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
+-{
+-	return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
+-}
+-
+-static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
+-						  unsigned int sb_index)
+-{
+-	struct devlink_sb *devlink_sb;
+-
+-	list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
+-		if (devlink_sb->index == sb_index)
+-			return devlink_sb;
+-	}
+-	return NULL;
+-}
+-
+-static bool devlink_sb_index_exists(struct devlink *devlink,
+-				    unsigned int sb_index)
+-{
+-	return devlink_sb_get_by_index(devlink, sb_index);
+-}
+-
+-static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
+-						    struct nlattr **attrs)
+-{
+-	if (attrs[DEVLINK_ATTR_SB_INDEX]) {
+-		u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
+-		struct devlink_sb *devlink_sb;
+-
+-		devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
+-		if (!devlink_sb)
+-			return ERR_PTR(-ENODEV);
+-		return devlink_sb;
+-	}
+-	return ERR_PTR(-EINVAL);
+-}
+-
+-static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
+-						   struct genl_info *info)
+-{
+-	return devlink_sb_get_from_attrs(devlink, info->attrs);
+-}
+-
+-static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
+-						struct nlattr **attrs,
+-						u16 *p_pool_index)
+-{
+-	u16 val;
+-
+-	if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
+-		return -EINVAL;
+-
+-	val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
+-	if (val >= devlink_sb_pool_count(devlink_sb))
+-		return -EINVAL;
+-	*p_pool_index = val;
+-	return 0;
+-}
+-
+-static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
+-					       struct genl_info *info,
+-					       u16 *p_pool_index)
+-{
+-	return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
+-						    p_pool_index);
+-}
+-
+-static int
+-devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
+-				    enum devlink_sb_pool_type *p_pool_type)
+-{
+-	u8 val;
+-
+-	if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
+-		return -EINVAL;
+-
+-	val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
+-	if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
+-	    val != DEVLINK_SB_POOL_TYPE_EGRESS)
+-		return -EINVAL;
+-	*p_pool_type = val;
+-	return 0;
+-}
+-
+-static int
+-devlink_sb_pool_type_get_from_info(struct genl_info *info,
+-				   enum devlink_sb_pool_type *p_pool_type)
+-{
+-	return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
+-}
+-
+-static int
+-devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
+-				  enum devlink_sb_threshold_type *p_th_type)
+-{
+-	u8 val;
+-
+-	if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
+-		return -EINVAL;
+-
+-	val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
+-	if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
+-	    val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
+-		return -EINVAL;
+-	*p_th_type = val;
+-	return 0;
+-}
+-
+-static int
+-devlink_sb_th_type_get_from_info(struct genl_info *info,
+-				 enum devlink_sb_threshold_type *p_th_type)
+-{
+-	return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
+-}
+-
+-static int
+-devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
+-				   struct nlattr **attrs,
+-				   enum devlink_sb_pool_type pool_type,
+-				   u16 *p_tc_index)
+-{
+-	u16 val;
+-
+-	if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
+-		return -EINVAL;
+-
+-	val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
+-	if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
+-	    val >= devlink_sb->ingress_tc_count)
+-		return -EINVAL;
+-	if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
+-	    val >= devlink_sb->egress_tc_count)
+-		return -EINVAL;
+-	*p_tc_index = val;
+-	return 0;
+-}
+-
+-static int
+-devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
+-				  struct genl_info *info,
+-				  enum devlink_sb_pool_type pool_type,
+-				  u16 *p_tc_index)
+-{
+-	return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
+-						  pool_type, p_tc_index);
+-}
+-
+-struct devlink_region {
+-	struct devlink *devlink;
+-	struct devlink_port *port;
+-	struct list_head list;
+-	union {
+-		const struct devlink_region_ops *ops;
+-		const struct devlink_port_region_ops *port_ops;
+-	};
+-	struct mutex snapshot_lock; /* protects snapshot_list,
+-				     * max_snapshots and cur_snapshots
+-				     * consistency.
+-				     */
+-	struct list_head snapshot_list;
+-	u32 max_snapshots;
+-	u32 cur_snapshots;
+-	u64 size;
+-};
+-
+-struct devlink_snapshot {
+-	struct list_head list;
+-	struct devlink_region *region;
+-	u8 *data;
+-	u32 id;
+-};
+-
+-static struct devlink_region *
+-devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
+-{
+-	struct devlink_region *region;
+-
+-	list_for_each_entry(region, &devlink->region_list, list)
+-		if (!strcmp(region->ops->name, region_name))
+-			return region;
+-
+-	return NULL;
+-}
+-
+-static struct devlink_region *
+-devlink_port_region_get_by_name(struct devlink_port *port,
+-				const char *region_name)
+-{
+-	struct devlink_region *region;
+-
+-	list_for_each_entry(region, &port->region_list, list)
+-		if (!strcmp(region->ops->name, region_name))
+-			return region;
+-
+-	return NULL;
+-}
+-
+-static struct devlink_snapshot *
+-devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
+-{
+-	struct devlink_snapshot *snapshot;
+-
+-	list_for_each_entry(snapshot, &region->snapshot_list, list)
+-		if (snapshot->id == id)
+-			return snapshot;
+-
+-	return NULL;
+-}
+-
+-#define DEVLINK_NL_FLAG_NEED_PORT		BIT(0)
+-#define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT	BIT(1)
+-#define DEVLINK_NL_FLAG_NEED_RATE		BIT(2)
+-#define DEVLINK_NL_FLAG_NEED_RATE_NODE		BIT(3)
+-#define DEVLINK_NL_FLAG_NEED_LINECARD		BIT(4)
+-
+-static int devlink_nl_pre_doit(const struct genl_ops *ops,
+-			       struct sk_buff *skb, struct genl_info *info)
+-{
+-	struct devlink_linecard *linecard;
+-	struct devlink_port *devlink_port;
+-	struct devlink *devlink;
+-	int err;
+-
+-	devlink = devlink_get_from_attrs(genl_info_net(info), info->attrs);
+-	if (IS_ERR(devlink))
+-		return PTR_ERR(devlink);
+-	devl_lock(devlink);
+-	info->user_ptr[0] = devlink;
+-	if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
+-		devlink_port = devlink_port_get_from_info(devlink, info);
+-		if (IS_ERR(devlink_port)) {
+-			err = PTR_ERR(devlink_port);
+-			goto unlock;
+-		}
+-		info->user_ptr[1] = devlink_port;
+-	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
+-		devlink_port = devlink_port_get_from_info(devlink, info);
+-		if (!IS_ERR(devlink_port))
+-			info->user_ptr[1] = devlink_port;
+-	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) {
+-		struct devlink_rate *devlink_rate;
+-
+-		devlink_rate = devlink_rate_get_from_info(devlink, info);
+-		if (IS_ERR(devlink_rate)) {
+-			err = PTR_ERR(devlink_rate);
+-			goto unlock;
+-		}
+-		info->user_ptr[1] = devlink_rate;
+-	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) {
+-		struct devlink_rate *rate_node;
+-
+-		rate_node = devlink_rate_node_get_from_info(devlink, info);
+-		if (IS_ERR(rate_node)) {
+-			err = PTR_ERR(rate_node);
+-			goto unlock;
+-		}
+-		info->user_ptr[1] = rate_node;
+-	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
+-		linecard = devlink_linecard_get_from_info(devlink, info);
+-		if (IS_ERR(linecard)) {
+-			err = PTR_ERR(linecard);
+-			goto unlock;
+-		}
+-		info->user_ptr[1] = linecard;
+-	}
+-	return 0;
+-
+-unlock:
+-	devl_unlock(devlink);
+-	devlink_put(devlink);
+-	return err;
+-}
+-
+-static void devlink_nl_post_doit(const struct genl_ops *ops,
+-				 struct sk_buff *skb, struct genl_info *info)
+-{
+-	struct devlink_linecard *linecard;
+-	struct devlink *devlink;
+-
+-	devlink = info->user_ptr[0];
+-	if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
+-		linecard = info->user_ptr[1];
+-		devlink_linecard_put(linecard);
+-	}
+-	devl_unlock(devlink);
+-	devlink_put(devlink);
+-}
+-
+-static struct genl_family devlink_nl_family;
+-
+-enum devlink_multicast_groups {
+-	DEVLINK_MCGRP_CONFIG,
+-};
+-
+-static const struct genl_multicast_group devlink_nl_mcgrps[] = {
+-	[DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
+-};
+-
+-static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
+-{
+-	if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
+-		return -EMSGSIZE;
+-	if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
+-		return -EMSGSIZE;
+-	return 0;
+-}
+-
+-static int devlink_nl_put_nested_handle(struct sk_buff *msg, struct devlink *devlink)
+-{
+-	struct nlattr *nested_attr;
+-
+-	nested_attr = nla_nest_start(msg, DEVLINK_ATTR_NESTED_DEVLINK);
+-	if (!nested_attr)
+-		return -EMSGSIZE;
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, nested_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, nested_attr);
+-	return -EMSGSIZE;
+-}
+-
+-struct devlink_reload_combination {
+-	enum devlink_reload_action action;
+-	enum devlink_reload_limit limit;
+-};
+-
+-static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
+-	{
+-		/* can't reinitialize driver with no down time */
+-		.action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
+-		.limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
+-	},
+-};
+-
+-static bool
+-devlink_reload_combination_is_invalid(enum devlink_reload_action action,
+-				      enum devlink_reload_limit limit)
+-{
+-	int i;
+-
+-	for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
+-		if (devlink_reload_invalid_combinations[i].action == action &&
+-		    devlink_reload_invalid_combinations[i].limit == limit)
+-			return true;
+-	return false;
+-}
+-
+-static bool
+-devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
+-{
+-	return test_bit(action, &devlink->ops->reload_actions);
+-}
+-
+-static bool
+-devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
+-{
+-	return test_bit(limit, &devlink->ops->reload_limits);
+-}
+-
+-static int devlink_reload_stat_put(struct sk_buff *msg,
+-				   enum devlink_reload_limit limit, u32 value)
+-{
+-	struct nlattr *reload_stats_entry;
+-
+-	reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
+-	if (!reload_stats_entry)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
+-	    nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
+-		goto nla_put_failure;
+-	nla_nest_end(msg, reload_stats_entry);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, reload_stats_entry);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
+-{
+-	struct nlattr *reload_stats_attr, *act_info, *act_stats;
+-	int i, j, stat_idx;
+-	u32 value;
+-
+-	if (!is_remote)
+-		reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
+-	else
+-		reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
+-
+-	if (!reload_stats_attr)
+-		return -EMSGSIZE;
+-
+-	for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
+-		if ((!is_remote &&
+-		     !devlink_reload_action_is_supported(devlink, i)) ||
+-		    i == DEVLINK_RELOAD_ACTION_UNSPEC)
+-			continue;
+-		act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
+-		if (!act_info)
+-			goto nla_put_failure;
+-
+-		if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
+-			goto action_info_nest_cancel;
+-		act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
+-		if (!act_stats)
+-			goto action_info_nest_cancel;
+-
+-		for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
+-			/* Remote stats are shown even if not locally supported.
+-			 * Stats of actions with unspecified limit are shown
+-			 * though drivers don't need to register unspecified
+-			 * limit.
+-			 */
+-			if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
+-			     !devlink_reload_limit_is_supported(devlink, j)) ||
+-			    devlink_reload_combination_is_invalid(i, j))
+-				continue;
+-
+-			stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
+-			if (!is_remote)
+-				value = devlink->stats.reload_stats[stat_idx];
+-			else
+-				value = devlink->stats.remote_reload_stats[stat_idx];
+-			if (devlink_reload_stat_put(msg, j, value))
+-				goto action_stats_nest_cancel;
+-		}
+-		nla_nest_end(msg, act_stats);
+-		nla_nest_end(msg, act_info);
+-	}
+-	nla_nest_end(msg, reload_stats_attr);
+-	return 0;
+-
+-action_stats_nest_cancel:
+-	nla_nest_cancel(msg, act_stats);
+-action_info_nest_cancel:
+-	nla_nest_cancel(msg, act_info);
+-nla_put_failure:
+-	nla_nest_cancel(msg, reload_stats_attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
+-			   enum devlink_command cmd, u32 portid,
+-			   u32 seq, int flags)
+-{
+-	struct nlattr *dev_stats;
+-	void *hdr;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
+-		goto nla_put_failure;
+-
+-	dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
+-	if (!dev_stats)
+-		goto nla_put_failure;
+-
+-	if (devlink_reload_stats_put(msg, devlink, false))
+-		goto dev_stats_nest_cancel;
+-	if (devlink_reload_stats_put(msg, devlink, true))
+-		goto dev_stats_nest_cancel;
+-
+-	nla_nest_end(msg, dev_stats);
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-dev_stats_nest_cancel:
+-	nla_nest_cancel(msg, dev_stats);
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
+-{
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
+-	WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int devlink_nl_port_attrs_put(struct sk_buff *msg,
+-				     struct devlink_port *devlink_port)
+-{
+-	struct devlink_port_attrs *attrs = &devlink_port->attrs;
+-
+-	if (!devlink_port->attrs_set)
+-		return 0;
+-	if (attrs->lanes) {
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
+-			return -EMSGSIZE;
+-	}
+-	if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
+-		return -EMSGSIZE;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
+-		return -EMSGSIZE;
+-	switch (devlink_port->attrs.flavour) {
+-	case DEVLINK_PORT_FLAVOUR_PCI_PF:
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
+-				attrs->pci_pf.controller) ||
+-		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
+-			return -EMSGSIZE;
+-		if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
+-			return -EMSGSIZE;
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_PCI_VF:
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
+-				attrs->pci_vf.controller) ||
+-		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
+-		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
+-			return -EMSGSIZE;
+-		if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
+-			return -EMSGSIZE;
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_PCI_SF:
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
+-				attrs->pci_sf.controller) ||
+-		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
+-				attrs->pci_sf.pf) ||
+-		    nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
+-				attrs->pci_sf.sf))
+-			return -EMSGSIZE;
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_PHYSICAL:
+-	case DEVLINK_PORT_FLAVOUR_CPU:
+-	case DEVLINK_PORT_FLAVOUR_DSA:
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
+-				attrs->phys.port_number))
+-			return -EMSGSIZE;
+-		if (!attrs->split)
+-			return 0;
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
+-				attrs->phys.port_number))
+-			return -EMSGSIZE;
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
+-				attrs->phys.split_subport_number))
+-			return -EMSGSIZE;
+-		break;
+-	default:
+-		break;
+-	}
+-	return 0;
+-}
+-
+-static int devlink_port_fn_hw_addr_fill(const struct devlink_ops *ops,
+-					struct devlink_port *port,
+-					struct sk_buff *msg,
+-					struct netlink_ext_ack *extack,
+-					bool *msg_updated)
+-{
+-	u8 hw_addr[MAX_ADDR_LEN];
+-	int hw_addr_len;
+-	int err;
+-
+-	if (!ops->port_function_hw_addr_get)
+-		return 0;
+-
+-	err = ops->port_function_hw_addr_get(port, hw_addr, &hw_addr_len,
+-					     extack);
+-	if (err) {
+-		if (err == -EOPNOTSUPP)
+-			return 0;
+-		return err;
+-	}
+-	err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
+-	if (err)
+-		return err;
+-	*msg_updated = true;
+-	return 0;
+-}
+-
+-static int devlink_nl_rate_fill(struct sk_buff *msg,
+-				struct devlink_rate *devlink_rate,
+-				enum devlink_command cmd, u32 portid, u32 seq,
+-				int flags, struct netlink_ext_ack *extack)
+-{
+-	struct devlink *devlink = devlink_rate->devlink;
+-	void *hdr;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type))
+-		goto nla_put_failure;
+-
+-	if (devlink_rate_is_leaf(devlink_rate)) {
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
+-				devlink_rate->devlink_port->index))
+-			goto nla_put_failure;
+-	} else if (devlink_rate_is_node(devlink_rate)) {
+-		if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME,
+-				   devlink_rate->name))
+-			goto nla_put_failure;
+-	}
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE,
+-			      devlink_rate->tx_share, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX,
+-			      devlink_rate->tx_max, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	if (devlink_rate->parent)
+-		if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
+-				   devlink_rate->parent->name))
+-			goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static bool
+-devlink_port_fn_state_valid(enum devlink_port_fn_state state)
+-{
+-	return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
+-	       state == DEVLINK_PORT_FN_STATE_ACTIVE;
+-}
+-
+-static bool
+-devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
+-{
+-	return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
+-	       opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
+-}
+-
+-static int devlink_port_fn_state_fill(const struct devlink_ops *ops,
+-				      struct devlink_port *port,
+-				      struct sk_buff *msg,
+-				      struct netlink_ext_ack *extack,
+-				      bool *msg_updated)
+-{
+-	enum devlink_port_fn_opstate opstate;
+-	enum devlink_port_fn_state state;
+-	int err;
+-
+-	if (!ops->port_fn_state_get)
+-		return 0;
+-
+-	err = ops->port_fn_state_get(port, &state, &opstate, extack);
+-	if (err) {
+-		if (err == -EOPNOTSUPP)
+-			return 0;
+-		return err;
+-	}
+-	if (!devlink_port_fn_state_valid(state)) {
+-		WARN_ON_ONCE(1);
+-		NL_SET_ERR_MSG_MOD(extack, "Invalid state read from driver");
+-		return -EINVAL;
+-	}
+-	if (!devlink_port_fn_opstate_valid(opstate)) {
+-		WARN_ON_ONCE(1);
+-		NL_SET_ERR_MSG_MOD(extack,
+-				   "Invalid operational state read from driver");
+-		return -EINVAL;
+-	}
+-	if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
+-	    nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
+-		return -EMSGSIZE;
+-	*msg_updated = true;
+-	return 0;
+-}
+-
+-static int
+-devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
+-				   struct netlink_ext_ack *extack)
+-{
+-	const struct devlink_ops *ops;
+-	struct nlattr *function_attr;
+-	bool msg_updated = false;
+-	int err;
+-
+-	function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
+-	if (!function_attr)
+-		return -EMSGSIZE;
+-
+-	ops = port->devlink->ops;
+-	err = devlink_port_fn_hw_addr_fill(ops, port, msg, extack,
+-					   &msg_updated);
+-	if (err)
+-		goto out;
+-	err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated);
+-out:
+-	if (err || !msg_updated)
+-		nla_nest_cancel(msg, function_attr);
+-	else
+-		nla_nest_end(msg, function_attr);
+-	return err;
+-}
+-
+-static int devlink_nl_port_fill(struct sk_buff *msg,
+-				struct devlink_port *devlink_port,
+-				enum devlink_command cmd, u32 portid, u32 seq,
+-				int flags, struct netlink_ext_ack *extack)
+-{
+-	struct devlink *devlink = devlink_port->devlink;
+-	void *hdr;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
+-		goto nla_put_failure;
+-
+-	/* Hold rtnl lock while accessing port's netdev attributes. */
+-	rtnl_lock();
+-	spin_lock_bh(&devlink_port->type_lock);
+-	if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
+-		goto nla_put_failure_type_locked;
+-	if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
+-	    nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
+-			devlink_port->desired_type))
+-		goto nla_put_failure_type_locked;
+-	if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
+-		struct net *net = devlink_net(devlink_port->devlink);
+-		struct net_device *netdev = devlink_port->type_dev;
+-
+-		if (netdev && net_eq(net, dev_net(netdev)) &&
+-		    (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
+-				 netdev->ifindex) ||
+-		     nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
+-				    netdev->name)))
+-			goto nla_put_failure_type_locked;
+-	}
+-	if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
+-		struct ib_device *ibdev = devlink_port->type_dev;
+-
+-		if (ibdev &&
+-		    nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
+-				   ibdev->name))
+-			goto nla_put_failure_type_locked;
+-	}
+-	spin_unlock_bh(&devlink_port->type_lock);
+-	rtnl_unlock();
+-	if (devlink_nl_port_attrs_put(msg, devlink_port))
+-		goto nla_put_failure;
+-	if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
+-		goto nla_put_failure;
+-	if (devlink_port->linecard &&
+-	    nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX,
+-			devlink_port->linecard->index))
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure_type_locked:
+-	spin_unlock_bh(&devlink_port->type_lock);
+-	rtnl_unlock();
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static void devlink_port_notify(struct devlink_port *devlink_port,
+-				enum devlink_command cmd)
+-{
+-	struct devlink *devlink = devlink_port->devlink;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
+-
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
+-				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static void devlink_rate_notify(struct devlink_rate *devlink_rate,
+-				enum devlink_command cmd)
+-{
+-	struct devlink *devlink = devlink_rate->devlink;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && cmd != DEVLINK_CMD_RATE_DEL);
+-
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_rate_fill(msg, devlink_rate, cmd, 0, 0, 0, NULL);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
+-				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg,
+-					  struct netlink_callback *cb)
+-{
+-	struct devlink_rate *devlink_rate;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
+-			enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
+-			u32 id = NETLINK_CB(cb->skb).portid;
+-
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
+-						   cb->nlh->nlmsg_seq,
+-						   NLM_F_MULTI, NULL);
+-			if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink_rate *devlink_rate = info->user_ptr[1];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
+-				   info->snd_portid, info->snd_seq, 0,
+-				   info->extack);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static bool
+-devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
+-			    struct devlink_rate *parent)
+-{
+-	while (parent) {
+-		if (parent == devlink_rate)
+-			return true;
+-		parent = parent->parent;
+-	}
+-	return false;
+-}
+-
+-static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
+-			      info->snd_portid, info->snd_seq, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
+-				     struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		if (idx < start) {
+-			idx++;
+-			devlink_put(devlink);
+-			continue;
+-		}
+-
+-		devl_lock(devlink);
+-		err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
+-				      NETLINK_CB(cb->skb).portid,
+-				      cb->nlh->nlmsg_seq, NLM_F_MULTI);
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-
+-		if (err)
+-			goto out;
+-		idx++;
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
+-				   info->snd_portid, info->snd_seq, 0,
+-				   info->extack);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
+-					  struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	struct devlink_port *devlink_port;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(devlink_port, &devlink->port_list, list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_port_fill(msg, devlink_port,
+-						   DEVLINK_CMD_NEW,
+-						   NETLINK_CB(cb->skb).portid,
+-						   cb->nlh->nlmsg_seq,
+-						   NLM_F_MULTI, cb->extack);
+-			if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_port_type_set(struct devlink_port *devlink_port,
+-				 enum devlink_port_type port_type)
+-
+-{
+-	int err;
+-
+-	if (!devlink_port->devlink->ops->port_type_set)
+-		return -EOPNOTSUPP;
+-
+-	if (port_type == devlink_port->type)
+-		return 0;
+-
+-	err = devlink_port->devlink->ops->port_type_set(devlink_port,
+-							port_type);
+-	if (err)
+-		return err;
+-
+-	devlink_port->desired_type = port_type;
+-	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
+-	return 0;
+-}
+-
+-static int devlink_port_function_hw_addr_set(struct devlink_port *port,
+-					     const struct nlattr *attr,
+-					     struct netlink_ext_ack *extack)
+-{
+-	const struct devlink_ops *ops = port->devlink->ops;
+-	const u8 *hw_addr;
+-	int hw_addr_len;
+-
+-	hw_addr = nla_data(attr);
+-	hw_addr_len = nla_len(attr);
+-	if (hw_addr_len > MAX_ADDR_LEN) {
+-		NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
+-		return -EINVAL;
+-	}
+-	if (port->type == DEVLINK_PORT_TYPE_ETH) {
+-		if (hw_addr_len != ETH_ALEN) {
+-			NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
+-			return -EINVAL;
+-		}
+-		if (!is_unicast_ether_addr(hw_addr)) {
+-			NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
+-			return -EINVAL;
+-		}
+-	}
+-
+-	if (!ops->port_function_hw_addr_set) {
+-		NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
+-		return -EOPNOTSUPP;
+-	}
+-
+-	return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len,
+-					      extack);
+-}
+-
+-static int devlink_port_fn_state_set(struct devlink_port *port,
+-				     const struct nlattr *attr,
+-				     struct netlink_ext_ack *extack)
+-{
+-	enum devlink_port_fn_state state;
+-	const struct devlink_ops *ops;
+-
+-	state = nla_get_u8(attr);
+-	ops = port->devlink->ops;
+-	if (!ops->port_fn_state_set) {
+-		NL_SET_ERR_MSG_MOD(extack,
+-				   "Function does not support state setting");
+-		return -EOPNOTSUPP;
+-	}
+-	return ops->port_fn_state_set(port, state, extack);
+-}
+-
+-static int devlink_port_function_set(struct devlink_port *port,
+-				     const struct nlattr *attr,
+-				     struct netlink_ext_ack *extack)
+-{
+-	struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
+-	int err;
+-
+-	err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
+-			       devlink_function_nl_policy, extack);
+-	if (err < 0) {
+-		NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
+-		return err;
+-	}
+-
+-	attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
+-	if (attr) {
+-		err = devlink_port_function_hw_addr_set(port, attr, extack);
+-		if (err)
+-			return err;
+-	}
+-	/* Keep this as the last function attribute set, so that when
+-	 * multiple port function attributes are set along with state,
+-	 * Those can be applied first before activating the state.
+-	 */
+-	attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
+-	if (attr)
+-		err = devlink_port_fn_state_set(port, attr, extack);
+-
+-	if (!err)
+-		devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	int err;
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
+-		enum devlink_port_type port_type;
+-
+-		port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
+-		err = devlink_port_type_set(devlink_port, port_type);
+-		if (err)
+-			return err;
+-	}
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
+-		struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
+-		struct netlink_ext_ack *extack = info->extack;
+-
+-		err = devlink_port_function_set(devlink_port, attr, extack);
+-		if (err)
+-			return err;
+-	}
+-
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
+-					  struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct devlink *devlink = info->user_ptr[0];
+-	u32 count;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_SPLIT_COUNT))
+-		return -EINVAL;
+-	if (!devlink->ops->port_split)
+-		return -EOPNOTSUPP;
+-
+-	count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
+-
+-	if (!devlink_port->attrs.splittable) {
+-		/* Split ports cannot be split. */
+-		if (devlink_port->attrs.split)
+-			NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
+-		else
+-			NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
+-		return -EINVAL;
+-	}
+-
+-	if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
+-		return -EINVAL;
+-	}
+-
+-	return devlink->ops->port_split(devlink, devlink_port, count,
+-					info->extack);
+-}
+-
+-static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
+-					    struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct devlink *devlink = info->user_ptr[0];
+-
+-	if (!devlink->ops->port_unsplit)
+-		return -EOPNOTSUPP;
+-	return devlink->ops->port_unsplit(devlink, devlink_port, info->extack);
+-}
+-
+-static int devlink_port_new_notify(struct devlink *devlink,
+-				   unsigned int port_index,
+-				   struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	lockdep_assert_held(&devlink->lock);
+-	devlink_port = devlink_port_get_by_index(devlink, port_index);
+-	if (!devlink_port) {
+-		err = -ENODEV;
+-		goto out;
+-	}
+-
+-	err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_NEW,
+-				   info->snd_portid, info->snd_seq, 0, NULL);
+-	if (err)
+-		goto out;
+-
+-	return genlmsg_reply(msg, info);
+-
+-out:
+-	nlmsg_free(msg);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_port_new_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink_port_new_attrs new_attrs = {};
+-	struct devlink *devlink = info->user_ptr[0];
+-	unsigned int new_port_index;
+-	int err;
+-
+-	if (!devlink->ops->port_new || !devlink->ops->port_del)
+-		return -EOPNOTSUPP;
+-
+-	if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
+-	    !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
+-		NL_SET_ERR_MSG_MOD(extack, "Port flavour or PCI PF are not specified");
+-		return -EINVAL;
+-	}
+-	new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
+-	new_attrs.pfnum =
+-		nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
+-		/* Port index of the new port being created by driver. */
+-		new_attrs.port_index =
+-			nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
+-		new_attrs.port_index_valid = true;
+-	}
+-	if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
+-		new_attrs.controller =
+-			nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
+-		new_attrs.controller_valid = true;
+-	}
+-	if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
+-	    info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
+-		new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
+-		new_attrs.sfnum_valid = true;
+-	}
+-
+-	err = devlink->ops->port_new(devlink, &new_attrs, extack,
+-				     &new_port_index);
+-	if (err)
+-		return err;
+-
+-	err = devlink_port_new_notify(devlink, new_port_index, info);
+-	if (err && err != -ENODEV) {
+-		/* Fail to send the response; destroy newly created port. */
+-		devlink->ops->port_del(devlink, new_port_index, extack);
+-	}
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-	unsigned int port_index;
+-
+-	if (!devlink->ops->port_del)
+-		return -EOPNOTSUPP;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_INDEX)) {
+-		NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
+-		return -EINVAL;
+-	}
+-	port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
+-
+-	return devlink->ops->port_del(devlink, port_index, extack);
+-}
+-
+-static int
+-devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
+-				struct genl_info *info,
+-				struct nlattr *nla_parent)
+-{
+-	struct devlink *devlink = devlink_rate->devlink;
+-	const char *parent_name = nla_data(nla_parent);
+-	const struct devlink_ops *ops = devlink->ops;
+-	size_t len = strlen(parent_name);
+-	struct devlink_rate *parent;
+-	int err = -EOPNOTSUPP;
+-
+-	parent = devlink_rate->parent;
+-	if (parent && len) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent.");
+-		return -EBUSY;
+-	} else if (parent && !len) {
+-		if (devlink_rate_is_leaf(devlink_rate))
+-			err = ops->rate_leaf_parent_set(devlink_rate, NULL,
+-							devlink_rate->priv, NULL,
+-							info->extack);
+-		else if (devlink_rate_is_node(devlink_rate))
+-			err = ops->rate_node_parent_set(devlink_rate, NULL,
+-							devlink_rate->priv, NULL,
+-							info->extack);
+-		if (err)
+-			return err;
+-
+-		refcount_dec(&parent->refcnt);
+-		devlink_rate->parent = NULL;
+-	} else if (!parent && len) {
+-		parent = devlink_rate_node_get_by_name(devlink, parent_name);
+-		if (IS_ERR(parent))
+-			return -ENODEV;
+-
+-		if (parent == devlink_rate) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed");
+-			return -EINVAL;
+-		}
+-
+-		if (devlink_rate_is_node(devlink_rate) &&
+-		    devlink_rate_is_parent_node(devlink_rate, parent->parent)) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node.");
+-			return -EEXIST;
+-		}
+-
+-		if (devlink_rate_is_leaf(devlink_rate))
+-			err = ops->rate_leaf_parent_set(devlink_rate, parent,
+-							devlink_rate->priv, parent->priv,
+-							info->extack);
+-		else if (devlink_rate_is_node(devlink_rate))
+-			err = ops->rate_node_parent_set(devlink_rate, parent,
+-							devlink_rate->priv, parent->priv,
+-							info->extack);
+-		if (err)
+-			return err;
+-
+-		refcount_inc(&parent->refcnt);
+-		devlink_rate->parent = parent;
+-	}
+-
+-	return 0;
+-}
+-
+-static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
+-			       const struct devlink_ops *ops,
+-			       struct genl_info *info)
+-{
+-	struct nlattr *nla_parent, **attrs = info->attrs;
+-	int err = -EOPNOTSUPP;
+-	u64 rate;
+-
+-	if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) {
+-		rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]);
+-		if (devlink_rate_is_leaf(devlink_rate))
+-			err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv,
+-							  rate, info->extack);
+-		else if (devlink_rate_is_node(devlink_rate))
+-			err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv,
+-							  rate, info->extack);
+-		if (err)
+-			return err;
+-		devlink_rate->tx_share = rate;
+-	}
+-
+-	if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) {
+-		rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]);
+-		if (devlink_rate_is_leaf(devlink_rate))
+-			err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv,
+-							rate, info->extack);
+-		else if (devlink_rate_is_node(devlink_rate))
+-			err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv,
+-							rate, info->extack);
+-		if (err)
+-			return err;
+-		devlink_rate->tx_max = rate;
+-	}
+-
+-	nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
+-	if (nla_parent) {
+-		err = devlink_nl_rate_parent_node_set(devlink_rate, info,
+-						      nla_parent);
+-		if (err)
+-			return err;
+-	}
+-
+-	return 0;
+-}
+-
+-static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
+-					   struct genl_info *info,
+-					   enum devlink_rate_type type)
+-{
+-	struct nlattr **attrs = info->attrs;
+-
+-	if (type == DEVLINK_RATE_TYPE_LEAF) {
+-		if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs");
+-			return false;
+-		}
+-		if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs");
+-			return false;
+-		}
+-		if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
+-		    !ops->rate_leaf_parent_set) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs");
+-			return false;
+-		}
+-	} else if (type == DEVLINK_RATE_TYPE_NODE) {
+-		if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes");
+-			return false;
+-		}
+-		if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes");
+-			return false;
+-		}
+-		if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
+-		    !ops->rate_node_parent_set) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes");
+-			return false;
+-		}
+-	} else {
+-		WARN(1, "Unknown type of rate object");
+-		return false;
+-	}
+-
+-	return true;
+-}
+-
+-static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink_rate *devlink_rate = info->user_ptr[1];
+-	struct devlink *devlink = devlink_rate->devlink;
+-	const struct devlink_ops *ops = devlink->ops;
+-	int err;
+-
+-	if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
+-		return -EOPNOTSUPP;
+-
+-	err = devlink_nl_rate_set(devlink_rate, ops, info);
+-
+-	if (!err)
+-		devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_rate *rate_node;
+-	const struct devlink_ops *ops;
+-	int err;
+-
+-	ops = devlink->ops;
+-	if (!ops || !ops->rate_node_new || !ops->rate_node_del) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported");
+-		return -EOPNOTSUPP;
+-	}
+-
+-	if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
+-		return -EOPNOTSUPP;
+-
+-	rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
+-	if (!IS_ERR(rate_node))
+-		return -EEXIST;
+-	else if (rate_node == ERR_PTR(-EINVAL))
+-		return -EINVAL;
+-
+-	rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL);
+-	if (!rate_node)
+-		return -ENOMEM;
+-
+-	rate_node->devlink = devlink;
+-	rate_node->type = DEVLINK_RATE_TYPE_NODE;
+-	rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL);
+-	if (!rate_node->name) {
+-		err = -ENOMEM;
+-		goto err_strdup;
+-	}
+-
+-	err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack);
+-	if (err)
+-		goto err_node_new;
+-
+-	err = devlink_nl_rate_set(rate_node, ops, info);
+-	if (err)
+-		goto err_rate_set;
+-
+-	refcount_set(&rate_node->refcnt, 1);
+-	list_add(&rate_node->list, &devlink->rate_list);
+-	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
+-	return 0;
+-
+-err_rate_set:
+-	ops->rate_node_del(rate_node, rate_node->priv, info->extack);
+-err_node_new:
+-	kfree(rate_node->name);
+-err_strdup:
+-	kfree(rate_node);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink_rate *rate_node = info->user_ptr[1];
+-	struct devlink *devlink = rate_node->devlink;
+-	const struct devlink_ops *ops = devlink->ops;
+-	int err;
+-
+-	if (refcount_read(&rate_node->refcnt) > 1) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node.");
+-		return -EBUSY;
+-	}
+-
+-	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
+-	err = ops->rate_node_del(rate_node, rate_node->priv, info->extack);
+-	if (rate_node->parent)
+-		refcount_dec(&rate_node->parent->refcnt);
+-	list_del(&rate_node->list);
+-	kfree(rate_node->name);
+-	kfree(rate_node);
+-	return err;
+-}
+-
+-struct devlink_linecard_type {
+-	const char *type;
+-	const void *priv;
+-};
+-
+-static int devlink_nl_linecard_fill(struct sk_buff *msg,
+-				    struct devlink *devlink,
+-				    struct devlink_linecard *linecard,
+-				    enum devlink_command cmd, u32 portid,
+-				    u32 seq, int flags,
+-				    struct netlink_ext_ack *extack)
+-{
+-	struct devlink_linecard_type *linecard_type;
+-	struct nlattr *attr;
+-	void *hdr;
+-	int i;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index))
+-		goto nla_put_failure;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state))
+-		goto nla_put_failure;
+-	if (linecard->type &&
+-	    nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, linecard->type))
+-		goto nla_put_failure;
+-
+-	if (linecard->types_count) {
+-		attr = nla_nest_start(msg,
+-				      DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES);
+-		if (!attr)
+-			goto nla_put_failure;
+-		for (i = 0; i < linecard->types_count; i++) {
+-			linecard_type = &linecard->types[i];
+-			if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE,
+-					   linecard_type->type)) {
+-				nla_nest_cancel(msg, attr);
+-				goto nla_put_failure;
+-			}
+-		}
+-		nla_nest_end(msg, attr);
+-	}
+-
+-	if (linecard->nested_devlink &&
+-	    devlink_nl_put_nested_handle(msg, linecard->nested_devlink))
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static void devlink_linecard_notify(struct devlink_linecard *linecard,
+-				    enum devlink_command cmd)
+-{
+-	struct devlink *devlink = linecard->devlink;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_LINECARD_NEW &&
+-		cmd != DEVLINK_CMD_LINECARD_DEL);
+-
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_linecard_fill(msg, devlink, linecard, cmd, 0, 0, 0,
+-				       NULL);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int devlink_nl_cmd_linecard_get_doit(struct sk_buff *skb,
+-					    struct genl_info *info)
+-{
+-	struct devlink_linecard *linecard = info->user_ptr[1];
+-	struct devlink *devlink = linecard->devlink;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	mutex_lock(&linecard->state_lock);
+-	err = devlink_nl_linecard_fill(msg, devlink, linecard,
+-				       DEVLINK_CMD_LINECARD_NEW,
+-				       info->snd_portid, info->snd_seq, 0,
+-				       info->extack);
+-	mutex_unlock(&linecard->state_lock);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg,
+-					      struct netlink_callback *cb)
+-{
+-	struct devlink_linecard *linecard;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		mutex_lock(&devlink->linecards_lock);
+-		list_for_each_entry(linecard, &devlink->linecard_list, list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			mutex_lock(&linecard->state_lock);
+-			err = devlink_nl_linecard_fill(msg, devlink, linecard,
+-						       DEVLINK_CMD_LINECARD_NEW,
+-						       NETLINK_CB(cb->skb).portid,
+-						       cb->nlh->nlmsg_seq,
+-						       NLM_F_MULTI,
+-						       cb->extack);
+-			mutex_unlock(&linecard->state_lock);
+-			if (err) {
+-				mutex_unlock(&devlink->linecards_lock);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		mutex_unlock(&devlink->linecards_lock);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static struct devlink_linecard_type *
+-devlink_linecard_type_lookup(struct devlink_linecard *linecard,
+-			     const char *type)
+-{
+-	struct devlink_linecard_type *linecard_type;
+-	int i;
+-
+-	for (i = 0; i < linecard->types_count; i++) {
+-		linecard_type = &linecard->types[i];
+-		if (!strcmp(type, linecard_type->type))
+-			return linecard_type;
+-	}
+-	return NULL;
+-}
+-
+-static int devlink_linecard_type_set(struct devlink_linecard *linecard,
+-				     const char *type,
+-				     struct netlink_ext_ack *extack)
+-{
+-	const struct devlink_linecard_ops *ops = linecard->ops;
+-	struct devlink_linecard_type *linecard_type;
+-	int err;
+-
+-	mutex_lock(&linecard->state_lock);
+-	if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
+-		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
+-		err = -EBUSY;
+-		goto out;
+-	}
+-	if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
+-		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
+-		err = -EBUSY;
+-		goto out;
+-	}
+-
+-	linecard_type = devlink_linecard_type_lookup(linecard, type);
+-	if (!linecard_type) {
+-		NL_SET_ERR_MSG_MOD(extack, "Unsupported line card type provided");
+-		err = -EINVAL;
+-		goto out;
+-	}
+-
+-	if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED &&
+-	    linecard->state != DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
+-		NL_SET_ERR_MSG_MOD(extack, "Line card already provisioned");
+-		err = -EBUSY;
+-		/* Check if the line card is provisioned in the same
+-		 * way the user asks. In case it is, make the operation
+-		 * to return success.
+-		 */
+-		if (ops->same_provision &&
+-		    ops->same_provision(linecard, linecard->priv,
+-					linecard_type->type,
+-					linecard_type->priv))
+-			err = 0;
+-		goto out;
+-	}
+-
+-	linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING;
+-	linecard->type = linecard_type->type;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-	err = ops->provision(linecard, linecard->priv, linecard_type->type,
+-			     linecard_type->priv, extack);
+-	if (err) {
+-		/* Provisioning failed. Assume the linecard is unprovisioned
+-		 * for future operations.
+-		 */
+-		mutex_lock(&linecard->state_lock);
+-		linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
+-		linecard->type = NULL;
+-		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-		mutex_unlock(&linecard->state_lock);
+-	}
+-	return err;
+-
+-out:
+-	mutex_unlock(&linecard->state_lock);
+-	return err;
+-}
+-
+-static int devlink_linecard_type_unset(struct devlink_linecard *linecard,
+-				       struct netlink_ext_ack *extack)
+-{
+-	int err;
+-
+-	mutex_lock(&linecard->state_lock);
+-	if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
+-		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
+-		err = -EBUSY;
+-		goto out;
+-	}
+-	if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
+-		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
+-		err = -EBUSY;
+-		goto out;
+-	}
+-	if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
+-		linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
+-		linecard->type = NULL;
+-		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-		err = 0;
+-		goto out;
+-	}
+-
+-	if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) {
+-		NL_SET_ERR_MSG_MOD(extack, "Line card is not provisioned");
+-		err = 0;
+-		goto out;
+-	}
+-	linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-	err = linecard->ops->unprovision(linecard, linecard->priv,
+-					 extack);
+-	if (err) {
+-		/* Unprovisioning failed. Assume the linecard is unprovisioned
+-		 * for future operations.
+-		 */
+-		mutex_lock(&linecard->state_lock);
+-		linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
+-		linecard->type = NULL;
+-		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-		mutex_unlock(&linecard->state_lock);
+-	}
+-	return err;
+-
+-out:
+-	mutex_unlock(&linecard->state_lock);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_linecard_set_doit(struct sk_buff *skb,
+-					    struct genl_info *info)
+-{
+-	struct devlink_linecard *linecard = info->user_ptr[1];
+-	struct netlink_ext_ack *extack = info->extack;
+-	int err;
+-
+-	if (info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) {
+-		const char *type;
+-
+-		type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]);
+-		if (strcmp(type, "")) {
+-			err = devlink_linecard_type_set(linecard, type, extack);
+-			if (err)
+-				return err;
+-		} else {
+-			err = devlink_linecard_type_unset(linecard, extack);
+-			if (err)
+-				return err;
+-		}
+-	}
+-
+-	return 0;
+-}
+-
+-static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
+-			      struct devlink_sb *devlink_sb,
+-			      enum devlink_command cmd, u32 portid,
+-			      u32 seq, int flags)
+-{
+-	void *hdr;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
+-			devlink_sb->ingress_pools_count))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
+-			devlink_sb->egress_pools_count))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
+-			devlink_sb->ingress_tc_count))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
+-			devlink_sb->egress_tc_count))
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
+-				      struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_sb *devlink_sb;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
+-				 DEVLINK_CMD_SB_NEW,
+-				 info->snd_portid, info->snd_seq, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
+-					struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	struct devlink_sb *devlink_sb;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
+-						 DEVLINK_CMD_SB_NEW,
+-						 NETLINK_CB(cb->skb).portid,
+-						 cb->nlh->nlmsg_seq,
+-						 NLM_F_MULTI);
+-			if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
+-				   struct devlink_sb *devlink_sb,
+-				   u16 pool_index, enum devlink_command cmd,
+-				   u32 portid, u32 seq, int flags)
+-{
+-	struct devlink_sb_pool_info pool_info;
+-	void *hdr;
+-	int err;
+-
+-	err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
+-					pool_index, &pool_info);
+-	if (err)
+-		return err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
+-		goto nla_put_failure;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
+-		goto nla_put_failure;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
+-		       pool_info.threshold_type))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
+-			pool_info.cell_size))
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
+-					   struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_sb *devlink_sb;
+-	struct sk_buff *msg;
+-	u16 pool_index;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
+-						  &pool_index);
+-	if (err)
+-		return err;
+-
+-	if (!devlink->ops->sb_pool_get)
+-		return -EOPNOTSUPP;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
+-				      DEVLINK_CMD_SB_POOL_NEW,
+-				      info->snd_portid, info->snd_seq, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
+-				struct devlink *devlink,
+-				struct devlink_sb *devlink_sb,
+-				u32 portid, u32 seq)
+-{
+-	u16 pool_count = devlink_sb_pool_count(devlink_sb);
+-	u16 pool_index;
+-	int err;
+-
+-	for (pool_index = 0; pool_index < pool_count; pool_index++) {
+-		if (*p_idx < start) {
+-			(*p_idx)++;
+-			continue;
+-		}
+-		err = devlink_nl_sb_pool_fill(msg, devlink,
+-					      devlink_sb,
+-					      pool_index,
+-					      DEVLINK_CMD_SB_POOL_NEW,
+-					      portid, seq, NLM_F_MULTI);
+-		if (err)
+-			return err;
+-		(*p_idx)++;
+-	}
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
+-					     struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	struct devlink_sb *devlink_sb;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		if (!devlink->ops->sb_pool_get)
+-			goto retry;
+-
+-		devl_lock(devlink);
+-		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
+-			err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
+-						   devlink_sb,
+-						   NETLINK_CB(cb->skb).portid,
+-						   cb->nlh->nlmsg_seq);
+-			if (err == -EOPNOTSUPP) {
+-				err = 0;
+-			} else if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-		}
+-		devl_unlock(devlink);
+-retry:
+-		devlink_put(devlink);
+-	}
+-out:
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
+-			       u16 pool_index, u32 size,
+-			       enum devlink_sb_threshold_type threshold_type,
+-			       struct netlink_ext_ack *extack)
+-
+-{
+-	const struct devlink_ops *ops = devlink->ops;
+-
+-	if (ops->sb_pool_set)
+-		return ops->sb_pool_set(devlink, sb_index, pool_index,
+-					size, threshold_type, extack);
+-	return -EOPNOTSUPP;
+-}
+-
+-static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
+-					   struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	enum devlink_sb_threshold_type threshold_type;
+-	struct devlink_sb *devlink_sb;
+-	u16 pool_index;
+-	u32 size;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
+-						  &pool_index);
+-	if (err)
+-		return err;
+-
+-	err = devlink_sb_th_type_get_from_info(info, &threshold_type);
+-	if (err)
+-		return err;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_POOL_SIZE))
+-		return -EINVAL;
+-
+-	size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
+-	return devlink_sb_pool_set(devlink, devlink_sb->index,
+-				   pool_index, size, threshold_type,
+-				   info->extack);
+-}
+-
+-static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
+-					struct devlink *devlink,
+-					struct devlink_port *devlink_port,
+-					struct devlink_sb *devlink_sb,
+-					u16 pool_index,
+-					enum devlink_command cmd,
+-					u32 portid, u32 seq, int flags)
+-{
+-	const struct devlink_ops *ops = devlink->ops;
+-	u32 threshold;
+-	void *hdr;
+-	int err;
+-
+-	err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
+-				    pool_index, &threshold);
+-	if (err)
+-		return err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
+-		goto nla_put_failure;
+-
+-	if (ops->sb_occ_port_pool_get) {
+-		u32 cur;
+-		u32 max;
+-
+-		err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
+-						pool_index, &cur, &max);
+-		if (err && err != -EOPNOTSUPP)
+-			goto sb_occ_get_failure;
+-		if (!err) {
+-			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
+-				goto nla_put_failure;
+-			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
+-				goto nla_put_failure;
+-		}
+-	}
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	err = -EMSGSIZE;
+-sb_occ_get_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
+-						struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct devlink *devlink = devlink_port->devlink;
+-	struct devlink_sb *devlink_sb;
+-	struct sk_buff *msg;
+-	u16 pool_index;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
+-						  &pool_index);
+-	if (err)
+-		return err;
+-
+-	if (!devlink->ops->sb_port_pool_get)
+-		return -EOPNOTSUPP;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
+-					   devlink_sb, pool_index,
+-					   DEVLINK_CMD_SB_PORT_POOL_NEW,
+-					   info->snd_portid, info->snd_seq, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
+-				     struct devlink *devlink,
+-				     struct devlink_sb *devlink_sb,
+-				     u32 portid, u32 seq)
+-{
+-	struct devlink_port *devlink_port;
+-	u16 pool_count = devlink_sb_pool_count(devlink_sb);
+-	u16 pool_index;
+-	int err;
+-
+-	list_for_each_entry(devlink_port, &devlink->port_list, list) {
+-		for (pool_index = 0; pool_index < pool_count; pool_index++) {
+-			if (*p_idx < start) {
+-				(*p_idx)++;
+-				continue;
+-			}
+-			err = devlink_nl_sb_port_pool_fill(msg, devlink,
+-							   devlink_port,
+-							   devlink_sb,
+-							   pool_index,
+-							   DEVLINK_CMD_SB_PORT_POOL_NEW,
+-							   portid, seq,
+-							   NLM_F_MULTI);
+-			if (err)
+-				return err;
+-			(*p_idx)++;
+-		}
+-	}
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
+-						  struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	struct devlink_sb *devlink_sb;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		if (!devlink->ops->sb_port_pool_get)
+-			goto retry;
+-
+-		devl_lock(devlink);
+-		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
+-			err = __sb_port_pool_get_dumpit(msg, start, &idx,
+-							devlink, devlink_sb,
+-							NETLINK_CB(cb->skb).portid,
+-							cb->nlh->nlmsg_seq);
+-			if (err == -EOPNOTSUPP) {
+-				err = 0;
+-			} else if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-		}
+-		devl_unlock(devlink);
+-retry:
+-		devlink_put(devlink);
+-	}
+-out:
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
+-				    unsigned int sb_index, u16 pool_index,
+-				    u32 threshold,
+-				    struct netlink_ext_ack *extack)
+-
+-{
+-	const struct devlink_ops *ops = devlink_port->devlink->ops;
+-
+-	if (ops->sb_port_pool_set)
+-		return ops->sb_port_pool_set(devlink_port, sb_index,
+-					     pool_index, threshold, extack);
+-	return -EOPNOTSUPP;
+-}
+-
+-static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
+-						struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_sb *devlink_sb;
+-	u16 pool_index;
+-	u32 threshold;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
+-						  &pool_index);
+-	if (err)
+-		return err;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD))
+-		return -EINVAL;
+-
+-	threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
+-	return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
+-					pool_index, threshold, info->extack);
+-}
+-
+-static int
+-devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
+-				struct devlink_port *devlink_port,
+-				struct devlink_sb *devlink_sb, u16 tc_index,
+-				enum devlink_sb_pool_type pool_type,
+-				enum devlink_command cmd,
+-				u32 portid, u32 seq, int flags)
+-{
+-	const struct devlink_ops *ops = devlink->ops;
+-	u16 pool_index;
+-	u32 threshold;
+-	void *hdr;
+-	int err;
+-
+-	err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
+-				       tc_index, pool_type,
+-				       &pool_index, &threshold);
+-	if (err)
+-		return err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
+-		goto nla_put_failure;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
+-		goto nla_put_failure;
+-	if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
+-		goto nla_put_failure;
+-	if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
+-		goto nla_put_failure;
+-
+-	if (ops->sb_occ_tc_port_bind_get) {
+-		u32 cur;
+-		u32 max;
+-
+-		err = ops->sb_occ_tc_port_bind_get(devlink_port,
+-						   devlink_sb->index,
+-						   tc_index, pool_type,
+-						   &cur, &max);
+-		if (err && err != -EOPNOTSUPP)
+-			return err;
+-		if (!err) {
+-			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
+-				goto nla_put_failure;
+-			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
+-				goto nla_put_failure;
+-		}
+-	}
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
+-						   struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct devlink *devlink = devlink_port->devlink;
+-	struct devlink_sb *devlink_sb;
+-	struct sk_buff *msg;
+-	enum devlink_sb_pool_type pool_type;
+-	u16 tc_index;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	err = devlink_sb_pool_type_get_from_info(info, &pool_type);
+-	if (err)
+-		return err;
+-
+-	err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
+-						pool_type, &tc_index);
+-	if (err)
+-		return err;
+-
+-	if (!devlink->ops->sb_tc_pool_bind_get)
+-		return -EOPNOTSUPP;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
+-					      devlink_sb, tc_index, pool_type,
+-					      DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
+-					      info->snd_portid,
+-					      info->snd_seq, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
+-					int start, int *p_idx,
+-					struct devlink *devlink,
+-					struct devlink_sb *devlink_sb,
+-					u32 portid, u32 seq)
+-{
+-	struct devlink_port *devlink_port;
+-	u16 tc_index;
+-	int err;
+-
+-	list_for_each_entry(devlink_port, &devlink->port_list, list) {
+-		for (tc_index = 0;
+-		     tc_index < devlink_sb->ingress_tc_count; tc_index++) {
+-			if (*p_idx < start) {
+-				(*p_idx)++;
+-				continue;
+-			}
+-			err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
+-							      devlink_port,
+-							      devlink_sb,
+-							      tc_index,
+-							      DEVLINK_SB_POOL_TYPE_INGRESS,
+-							      DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
+-							      portid, seq,
+-							      NLM_F_MULTI);
+-			if (err)
+-				return err;
+-			(*p_idx)++;
+-		}
+-		for (tc_index = 0;
+-		     tc_index < devlink_sb->egress_tc_count; tc_index++) {
+-			if (*p_idx < start) {
+-				(*p_idx)++;
+-				continue;
+-			}
+-			err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
+-							      devlink_port,
+-							      devlink_sb,
+-							      tc_index,
+-							      DEVLINK_SB_POOL_TYPE_EGRESS,
+-							      DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
+-							      portid, seq,
+-							      NLM_F_MULTI);
+-			if (err)
+-				return err;
+-			(*p_idx)++;
+-		}
+-	}
+-	return 0;
+-}
+-
+-static int
+-devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
+-					  struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	struct devlink_sb *devlink_sb;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		if (!devlink->ops->sb_tc_pool_bind_get)
+-			goto retry;
+-
+-		devl_lock(devlink);
+-		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
+-			err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
+-							   devlink,
+-							   devlink_sb,
+-							   NETLINK_CB(cb->skb).portid,
+-							   cb->nlh->nlmsg_seq);
+-			if (err == -EOPNOTSUPP) {
+-				err = 0;
+-			} else if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-		}
+-		devl_unlock(devlink);
+-retry:
+-		devlink_put(devlink);
+-	}
+-out:
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
+-				       unsigned int sb_index, u16 tc_index,
+-				       enum devlink_sb_pool_type pool_type,
+-				       u16 pool_index, u32 threshold,
+-				       struct netlink_ext_ack *extack)
+-
+-{
+-	const struct devlink_ops *ops = devlink_port->devlink->ops;
+-
+-	if (ops->sb_tc_pool_bind_set)
+-		return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
+-						tc_index, pool_type,
+-						pool_index, threshold, extack);
+-	return -EOPNOTSUPP;
+-}
+-
+-static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
+-						   struct genl_info *info)
+-{
+-	struct devlink_port *devlink_port = info->user_ptr[1];
+-	struct devlink *devlink = info->user_ptr[0];
+-	enum devlink_sb_pool_type pool_type;
+-	struct devlink_sb *devlink_sb;
+-	u16 tc_index;
+-	u16 pool_index;
+-	u32 threshold;
+-	int err;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	err = devlink_sb_pool_type_get_from_info(info, &pool_type);
+-	if (err)
+-		return err;
+-
+-	err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
+-						pool_type, &tc_index);
+-	if (err)
+-		return err;
+-
+-	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
+-						  &pool_index);
+-	if (err)
+-		return err;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD))
+-		return -EINVAL;
+-
+-	threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
+-	return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
+-					   tc_index, pool_type,
+-					   pool_index, threshold, info->extack);
+-}
+-
+-static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
+-					       struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	const struct devlink_ops *ops = devlink->ops;
+-	struct devlink_sb *devlink_sb;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	if (ops->sb_occ_snapshot)
+-		return ops->sb_occ_snapshot(devlink, devlink_sb->index);
+-	return -EOPNOTSUPP;
+-}
+-
+-static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
+-						struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	const struct devlink_ops *ops = devlink->ops;
+-	struct devlink_sb *devlink_sb;
+-
+-	devlink_sb = devlink_sb_get_from_info(devlink, info);
+-	if (IS_ERR(devlink_sb))
+-		return PTR_ERR(devlink_sb);
+-
+-	if (ops->sb_occ_max_clear)
+-		return ops->sb_occ_max_clear(devlink, devlink_sb->index);
+-	return -EOPNOTSUPP;
+-}
+-
+-static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
+-				   enum devlink_command cmd, u32 portid,
+-				   u32 seq, int flags)
+-{
+-	const struct devlink_ops *ops = devlink->ops;
+-	enum devlink_eswitch_encap_mode encap_mode;
+-	u8 inline_mode;
+-	void *hdr;
+-	int err = 0;
+-	u16 mode;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	err = devlink_nl_put_handle(msg, devlink);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	if (ops->eswitch_mode_get) {
+-		err = ops->eswitch_mode_get(devlink, &mode);
+-		if (err)
+-			goto nla_put_failure;
+-		err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
+-		if (err)
+-			goto nla_put_failure;
+-	}
+-
+-	if (ops->eswitch_inline_mode_get) {
+-		err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
+-		if (err)
+-			goto nla_put_failure;
+-		err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
+-				 inline_mode);
+-		if (err)
+-			goto nla_put_failure;
+-	}
+-
+-	if (ops->eswitch_encap_mode_get) {
+-		err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
+-		if (err)
+-			goto nla_put_failure;
+-		err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
+-		if (err)
+-			goto nla_put_failure;
+-	}
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
+-					   struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
+-				      info->snd_portid, info->snd_seq, 0);
+-
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode,
+-				    struct netlink_ext_ack *extack)
+-{
+-	struct devlink_rate *devlink_rate;
+-
+-	list_for_each_entry(devlink_rate, &devlink->rate_list, list)
+-		if (devlink_rate_is_node(devlink_rate)) {
+-			NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists.");
+-			return -EBUSY;
+-		}
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
+-					   struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	const struct devlink_ops *ops = devlink->ops;
+-	enum devlink_eswitch_encap_mode encap_mode;
+-	u8 inline_mode;
+-	int err = 0;
+-	u16 mode;
+-
+-	if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
+-		if (!ops->eswitch_mode_set)
+-			return -EOPNOTSUPP;
+-		mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
+-		err = devlink_rate_nodes_check(devlink, mode, info->extack);
+-		if (err)
+-			return err;
+-		err = ops->eswitch_mode_set(devlink, mode, info->extack);
+-		if (err)
+-			return err;
+-	}
+-
+-	if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
+-		if (!ops->eswitch_inline_mode_set)
+-			return -EOPNOTSUPP;
+-		inline_mode = nla_get_u8(
+-				info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
+-		err = ops->eswitch_inline_mode_set(devlink, inline_mode,
+-						   info->extack);
+-		if (err)
+-			return err;
+-	}
+-
+-	if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
+-		if (!ops->eswitch_encap_mode_set)
+-			return -EOPNOTSUPP;
+-		encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
+-		err = ops->eswitch_encap_mode_set(devlink, encap_mode,
+-						  info->extack);
+-		if (err)
+-			return err;
+-	}
+-
+-	return 0;
+-}
+-
+-int devlink_dpipe_match_put(struct sk_buff *skb,
+-			    struct devlink_dpipe_match *match)
+-{
+-	struct devlink_dpipe_header *header = match->header;
+-	struct devlink_dpipe_field *field = &header->fields[match->field_id];
+-	struct nlattr *match_attr;
+-
+-	match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
+-	if (!match_attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
+-	    nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(skb, match_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, match_attr);
+-	return -EMSGSIZE;
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
+-
+-static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
+-				     struct sk_buff *skb)
+-{
+-	struct nlattr *matches_attr;
+-
+-	matches_attr = nla_nest_start_noflag(skb,
+-					     DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
+-	if (!matches_attr)
+-		return -EMSGSIZE;
+-
+-	if (table->table_ops->matches_dump(table->priv, skb))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(skb, matches_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, matches_attr);
+-	return -EMSGSIZE;
+-}
+-
+-int devlink_dpipe_action_put(struct sk_buff *skb,
+-			     struct devlink_dpipe_action *action)
+-{
+-	struct devlink_dpipe_header *header = action->header;
+-	struct devlink_dpipe_field *field = &header->fields[action->field_id];
+-	struct nlattr *action_attr;
+-
+-	action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
+-	if (!action_attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
+-	    nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(skb, action_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, action_attr);
+-	return -EMSGSIZE;
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
+-
+-static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
+-				     struct sk_buff *skb)
+-{
+-	struct nlattr *actions_attr;
+-
+-	actions_attr = nla_nest_start_noflag(skb,
+-					     DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
+-	if (!actions_attr)
+-		return -EMSGSIZE;
+-
+-	if (table->table_ops->actions_dump(table->priv, skb))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(skb, actions_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, actions_attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_dpipe_table_put(struct sk_buff *skb,
+-				   struct devlink_dpipe_table *table)
+-{
+-	struct nlattr *table_attr;
+-	u64 table_size;
+-
+-	table_size = table->table_ops->size_get(table->priv);
+-	table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
+-	if (!table_attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
+-	    nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-	if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
+-		       table->counters_enabled))
+-		goto nla_put_failure;
+-
+-	if (table->resource_valid) {
+-		if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
+-				      table->resource_id, DEVLINK_ATTR_PAD) ||
+-		    nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
+-				      table->resource_units, DEVLINK_ATTR_PAD))
+-			goto nla_put_failure;
+-	}
+-	if (devlink_dpipe_matches_put(table, skb))
+-		goto nla_put_failure;
+-
+-	if (devlink_dpipe_actions_put(table, skb))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(skb, table_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, table_attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
+-					    struct genl_info *info)
+-{
+-	int err;
+-
+-	if (*pskb) {
+-		err = genlmsg_reply(*pskb, info);
+-		if (err)
+-			return err;
+-	}
+-	*pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!*pskb)
+-		return -ENOMEM;
+-	return 0;
+-}
+-
+-static int devlink_dpipe_tables_fill(struct genl_info *info,
+-				     enum devlink_command cmd, int flags,
+-				     struct list_head *dpipe_tables,
+-				     const char *table_name)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_dpipe_table *table;
+-	struct nlattr *tables_attr;
+-	struct sk_buff *skb = NULL;
+-	struct nlmsghdr *nlh;
+-	bool incomplete;
+-	void *hdr;
+-	int i;
+-	int err;
+-
+-	table = list_first_entry(dpipe_tables,
+-				 struct devlink_dpipe_table, list);
+-start_again:
+-	err = devlink_dpipe_send_and_alloc_skb(&skb, info);
+-	if (err)
+-		return err;
+-
+-	hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			  &devlink_nl_family, NLM_F_MULTI, cmd);
+-	if (!hdr) {
+-		nlmsg_free(skb);
+-		return -EMSGSIZE;
+-	}
+-
+-	if (devlink_nl_put_handle(skb, devlink))
+-		goto nla_put_failure;
+-	tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
+-	if (!tables_attr)
+-		goto nla_put_failure;
+-
+-	i = 0;
+-	incomplete = false;
+-	list_for_each_entry_from(table, dpipe_tables, list) {
+-		if (!table_name) {
+-			err = devlink_dpipe_table_put(skb, table);
+-			if (err) {
+-				if (!i)
+-					goto err_table_put;
+-				incomplete = true;
+-				break;
+-			}
+-		} else {
+-			if (!strcmp(table->name, table_name)) {
+-				err = devlink_dpipe_table_put(skb, table);
+-				if (err)
+-					break;
+-			}
+-		}
+-		i++;
+-	}
+-
+-	nla_nest_end(skb, tables_attr);
+-	genlmsg_end(skb, hdr);
+-	if (incomplete)
+-		goto start_again;
+-
+-send_done:
+-	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			NLMSG_DONE, 0, flags | NLM_F_MULTI);
+-	if (!nlh) {
+-		err = devlink_dpipe_send_and_alloc_skb(&skb, info);
+-		if (err)
+-			return err;
+-		goto send_done;
+-	}
+-
+-	return genlmsg_reply(skb, info);
+-
+-nla_put_failure:
+-	err = -EMSGSIZE;
+-err_table_put:
+-	nlmsg_free(skb);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
+-					  struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	const char *table_name =  NULL;
+-
+-	if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
+-		table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
+-
+-	return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
+-					 &devlink->dpipe_table_list,
+-					 table_name);
+-}
+-
+-static int devlink_dpipe_value_put(struct sk_buff *skb,
+-				   struct devlink_dpipe_value *value)
+-{
+-	if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
+-		    value->value_size, value->value))
+-		return -EMSGSIZE;
+-	if (value->mask)
+-		if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
+-			    value->value_size, value->mask))
+-			return -EMSGSIZE;
+-	if (value->mapping_valid)
+-		if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
+-				value->mapping_value))
+-			return -EMSGSIZE;
+-	return 0;
+-}
+-
+-static int devlink_dpipe_action_value_put(struct sk_buff *skb,
+-					  struct devlink_dpipe_value *value)
+-{
+-	if (!value->action)
+-		return -EINVAL;
+-	if (devlink_dpipe_action_put(skb, value->action))
+-		return -EMSGSIZE;
+-	if (devlink_dpipe_value_put(skb, value))
+-		return -EMSGSIZE;
+-	return 0;
+-}
+-
+-static int devlink_dpipe_action_values_put(struct sk_buff *skb,
+-					   struct devlink_dpipe_value *values,
+-					   unsigned int values_count)
+-{
+-	struct nlattr *action_attr;
+-	int i;
+-	int err;
+-
+-	for (i = 0; i < values_count; i++) {
+-		action_attr = nla_nest_start_noflag(skb,
+-						    DEVLINK_ATTR_DPIPE_ACTION_VALUE);
+-		if (!action_attr)
+-			return -EMSGSIZE;
+-		err = devlink_dpipe_action_value_put(skb, &values[i]);
+-		if (err)
+-			goto err_action_value_put;
+-		nla_nest_end(skb, action_attr);
+-	}
+-	return 0;
+-
+-err_action_value_put:
+-	nla_nest_cancel(skb, action_attr);
+-	return err;
+-}
+-
+-static int devlink_dpipe_match_value_put(struct sk_buff *skb,
+-					 struct devlink_dpipe_value *value)
+-{
+-	if (!value->match)
+-		return -EINVAL;
+-	if (devlink_dpipe_match_put(skb, value->match))
+-		return -EMSGSIZE;
+-	if (devlink_dpipe_value_put(skb, value))
+-		return -EMSGSIZE;
+-	return 0;
+-}
+-
+-static int devlink_dpipe_match_values_put(struct sk_buff *skb,
+-					  struct devlink_dpipe_value *values,
+-					  unsigned int values_count)
+-{
+-	struct nlattr *match_attr;
+-	int i;
+-	int err;
+-
+-	for (i = 0; i < values_count; i++) {
+-		match_attr = nla_nest_start_noflag(skb,
+-						   DEVLINK_ATTR_DPIPE_MATCH_VALUE);
+-		if (!match_attr)
+-			return -EMSGSIZE;
+-		err = devlink_dpipe_match_value_put(skb, &values[i]);
+-		if (err)
+-			goto err_match_value_put;
+-		nla_nest_end(skb, match_attr);
+-	}
+-	return 0;
+-
+-err_match_value_put:
+-	nla_nest_cancel(skb, match_attr);
+-	return err;
+-}
+-
+-static int devlink_dpipe_entry_put(struct sk_buff *skb,
+-				   struct devlink_dpipe_entry *entry)
+-{
+-	struct nlattr *entry_attr, *matches_attr, *actions_attr;
+-	int err;
+-
+-	entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
+-	if (!entry_attr)
+-		return  -EMSGSIZE;
+-
+-	if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-	if (entry->counter_valid)
+-		if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
+-				      entry->counter, DEVLINK_ATTR_PAD))
+-			goto nla_put_failure;
+-
+-	matches_attr = nla_nest_start_noflag(skb,
+-					     DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
+-	if (!matches_attr)
+-		goto nla_put_failure;
+-
+-	err = devlink_dpipe_match_values_put(skb, entry->match_values,
+-					     entry->match_values_count);
+-	if (err) {
+-		nla_nest_cancel(skb, matches_attr);
+-		goto err_match_values_put;
+-	}
+-	nla_nest_end(skb, matches_attr);
+-
+-	actions_attr = nla_nest_start_noflag(skb,
+-					     DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
+-	if (!actions_attr)
+-		goto nla_put_failure;
+-
+-	err = devlink_dpipe_action_values_put(skb, entry->action_values,
+-					      entry->action_values_count);
+-	if (err) {
+-		nla_nest_cancel(skb, actions_attr);
+-		goto err_action_values_put;
+-	}
+-	nla_nest_end(skb, actions_attr);
+-
+-	nla_nest_end(skb, entry_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	err = -EMSGSIZE;
+-err_match_values_put:
+-err_action_values_put:
+-	nla_nest_cancel(skb, entry_attr);
+-	return err;
+-}
+-
+-static struct devlink_dpipe_table *
+-devlink_dpipe_table_find(struct list_head *dpipe_tables,
+-			 const char *table_name, struct devlink *devlink)
+-{
+-	struct devlink_dpipe_table *table;
+-	list_for_each_entry_rcu(table, dpipe_tables, list,
+-				lockdep_is_held(&devlink->lock)) {
+-		if (!strcmp(table->name, table_name))
+-			return table;
+-	}
+-	return NULL;
+-}
+-
+-int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
+-{
+-	struct devlink *devlink;
+-	int err;
+-
+-	err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
+-					       dump_ctx->info);
+-	if (err)
+-		return err;
+-
+-	dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
+-				    dump_ctx->info->snd_portid,
+-				    dump_ctx->info->snd_seq,
+-				    &devlink_nl_family, NLM_F_MULTI,
+-				    dump_ctx->cmd);
+-	if (!dump_ctx->hdr)
+-		goto nla_put_failure;
+-
+-	devlink = dump_ctx->info->user_ptr[0];
+-	if (devlink_nl_put_handle(dump_ctx->skb, devlink))
+-		goto nla_put_failure;
+-	dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
+-					       DEVLINK_ATTR_DPIPE_ENTRIES);
+-	if (!dump_ctx->nest)
+-		goto nla_put_failure;
+-	return 0;
+-
+-nla_put_failure:
+-	nlmsg_free(dump_ctx->skb);
+-	return -EMSGSIZE;
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
+-
+-int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
+-				   struct devlink_dpipe_entry *entry)
+-{
+-	return devlink_dpipe_entry_put(dump_ctx->skb, entry);
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
+-
+-int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
+-{
+-	nla_nest_end(dump_ctx->skb, dump_ctx->nest);
+-	genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
+-
+-void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
+-
+-{
+-	unsigned int value_count, value_index;
+-	struct devlink_dpipe_value *value;
+-
+-	value = entry->action_values;
+-	value_count = entry->action_values_count;
+-	for (value_index = 0; value_index < value_count; value_index++) {
+-		kfree(value[value_index].value);
+-		kfree(value[value_index].mask);
+-	}
+-
+-	value = entry->match_values;
+-	value_count = entry->match_values_count;
+-	for (value_index = 0; value_index < value_count; value_index++) {
+-		kfree(value[value_index].value);
+-		kfree(value[value_index].mask);
+-	}
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_entry_clear);
+-
+-static int devlink_dpipe_entries_fill(struct genl_info *info,
+-				      enum devlink_command cmd, int flags,
+-				      struct devlink_dpipe_table *table)
+-{
+-	struct devlink_dpipe_dump_ctx dump_ctx;
+-	struct nlmsghdr *nlh;
+-	int err;
+-
+-	dump_ctx.skb = NULL;
+-	dump_ctx.cmd = cmd;
+-	dump_ctx.info = info;
+-
+-	err = table->table_ops->entries_dump(table->priv,
+-					     table->counters_enabled,
+-					     &dump_ctx);
+-	if (err)
+-		return err;
+-
+-send_done:
+-	nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
+-			NLMSG_DONE, 0, flags | NLM_F_MULTI);
+-	if (!nlh) {
+-		err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
+-		if (err)
+-			return err;
+-		goto send_done;
+-	}
+-	return genlmsg_reply(dump_ctx.skb, info);
+-}
+-
+-static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
+-					    struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_dpipe_table *table;
+-	const char *table_name;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME))
+-		return -EINVAL;
+-
+-	table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
+-	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
+-					 table_name, devlink);
+-	if (!table)
+-		return -EINVAL;
+-
+-	if (!table->table_ops->entries_dump)
+-		return -EINVAL;
+-
+-	return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
+-					  0, table);
+-}
+-
+-static int devlink_dpipe_fields_put(struct sk_buff *skb,
+-				    const struct devlink_dpipe_header *header)
+-{
+-	struct devlink_dpipe_field *field;
+-	struct nlattr *field_attr;
+-	int i;
+-
+-	for (i = 0; i < header->fields_count; i++) {
+-		field = &header->fields[i];
+-		field_attr = nla_nest_start_noflag(skb,
+-						   DEVLINK_ATTR_DPIPE_FIELD);
+-		if (!field_attr)
+-			return -EMSGSIZE;
+-		if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
+-		    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
+-		    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
+-		    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
+-			goto nla_put_failure;
+-		nla_nest_end(skb, field_attr);
+-	}
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, field_attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_dpipe_header_put(struct sk_buff *skb,
+-				    struct devlink_dpipe_header *header)
+-{
+-	struct nlattr *fields_attr, *header_attr;
+-	int err;
+-
+-	header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
+-	if (!header_attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
+-	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
+-	    nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
+-		goto nla_put_failure;
+-
+-	fields_attr = nla_nest_start_noflag(skb,
+-					    DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
+-	if (!fields_attr)
+-		goto nla_put_failure;
+-
+-	err = devlink_dpipe_fields_put(skb, header);
+-	if (err) {
+-		nla_nest_cancel(skb, fields_attr);
+-		goto nla_put_failure;
+-	}
+-	nla_nest_end(skb, fields_attr);
+-	nla_nest_end(skb, header_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	err = -EMSGSIZE;
+-	nla_nest_cancel(skb, header_attr);
+-	return err;
+-}
+-
+-static int devlink_dpipe_headers_fill(struct genl_info *info,
+-				      enum devlink_command cmd, int flags,
+-				      struct devlink_dpipe_headers *
+-				      dpipe_headers)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct nlattr *headers_attr;
+-	struct sk_buff *skb = NULL;
+-	struct nlmsghdr *nlh;
+-	void *hdr;
+-	int i, j;
+-	int err;
+-
+-	i = 0;
+-start_again:
+-	err = devlink_dpipe_send_and_alloc_skb(&skb, info);
+-	if (err)
+-		return err;
+-
+-	hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			  &devlink_nl_family, NLM_F_MULTI, cmd);
+-	if (!hdr) {
+-		nlmsg_free(skb);
+-		return -EMSGSIZE;
+-	}
+-
+-	if (devlink_nl_put_handle(skb, devlink))
+-		goto nla_put_failure;
+-	headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
+-	if (!headers_attr)
+-		goto nla_put_failure;
+-
+-	j = 0;
+-	for (; i < dpipe_headers->headers_count; i++) {
+-		err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
+-		if (err) {
+-			if (!j)
+-				goto err_table_put;
+-			break;
+-		}
+-		j++;
+-	}
+-	nla_nest_end(skb, headers_attr);
+-	genlmsg_end(skb, hdr);
+-	if (i != dpipe_headers->headers_count)
+-		goto start_again;
+-
+-send_done:
+-	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			NLMSG_DONE, 0, flags | NLM_F_MULTI);
+-	if (!nlh) {
+-		err = devlink_dpipe_send_and_alloc_skb(&skb, info);
+-		if (err)
+-			return err;
+-		goto send_done;
+-	}
+-	return genlmsg_reply(skb, info);
+-
+-nla_put_failure:
+-	err = -EMSGSIZE;
+-err_table_put:
+-	nlmsg_free(skb);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
+-					    struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-
+-	if (!devlink->dpipe_headers)
+-		return -EOPNOTSUPP;
+-	return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
+-					  0, devlink->dpipe_headers);
+-}
+-
+-static int devlink_dpipe_table_counters_set(struct devlink *devlink,
+-					    const char *table_name,
+-					    bool enable)
+-{
+-	struct devlink_dpipe_table *table;
+-
+-	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
+-					 table_name, devlink);
+-	if (!table)
+-		return -EINVAL;
+-
+-	if (table->counter_control_extern)
+-		return -EOPNOTSUPP;
+-
+-	if (!(table->counters_enabled ^ enable))
+-		return 0;
+-
+-	table->counters_enabled = enable;
+-	if (table->table_ops->counters_set_update)
+-		table->table_ops->counters_set_update(table->priv, enable);
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
+-						   struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	const char *table_name;
+-	bool counters_enable;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME) ||
+-	    GENL_REQ_ATTR_CHECK(info,
+-				DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED))
+-		return -EINVAL;
+-
+-	table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
+-	counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
+-
+-	return devlink_dpipe_table_counters_set(devlink, table_name,
+-						counters_enable);
+-}
+-
+-static struct devlink_resource *
+-devlink_resource_find(struct devlink *devlink,
+-		      struct devlink_resource *resource, u64 resource_id)
+-{
+-	struct list_head *resource_list;
+-
+-	if (resource)
+-		resource_list = &resource->resource_list;
+-	else
+-		resource_list = &devlink->resource_list;
+-
+-	list_for_each_entry(resource, resource_list, list) {
+-		struct devlink_resource *child_resource;
+-
+-		if (resource->id == resource_id)
+-			return resource;
+-
+-		child_resource = devlink_resource_find(devlink, resource,
+-						       resource_id);
+-		if (child_resource)
+-			return child_resource;
+-	}
+-	return NULL;
+-}
+-
+-static void
+-devlink_resource_validate_children(struct devlink_resource *resource)
+-{
+-	struct devlink_resource *child_resource;
+-	bool size_valid = true;
+-	u64 parts_size = 0;
+-
+-	if (list_empty(&resource->resource_list))
+-		goto out;
+-
+-	list_for_each_entry(child_resource, &resource->resource_list, list)
+-		parts_size += child_resource->size_new;
+-
+-	if (parts_size > resource->size_new)
+-		size_valid = false;
+-out:
+-	resource->size_valid = size_valid;
+-}
+-
+-static int
+-devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
+-			       struct netlink_ext_ack *extack)
+-{
+-	u64 reminder;
+-	int err = 0;
+-
+-	if (size > resource->size_params.size_max) {
+-		NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
+-		err = -EINVAL;
+-	}
+-
+-	if (size < resource->size_params.size_min) {
+-		NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
+-		err = -EINVAL;
+-	}
+-
+-	div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
+-	if (reminder) {
+-		NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
+-		err = -EINVAL;
+-	}
+-
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
+-				       struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_resource *resource;
+-	u64 resource_id;
+-	u64 size;
+-	int err;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_ID) ||
+-	    GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_SIZE))
+-		return -EINVAL;
+-	resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
+-
+-	resource = devlink_resource_find(devlink, NULL, resource_id);
+-	if (!resource)
+-		return -EINVAL;
+-
+-	size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
+-	err = devlink_resource_validate_size(resource, size, info->extack);
+-	if (err)
+-		return err;
+-
+-	resource->size_new = size;
+-	devlink_resource_validate_children(resource);
+-	if (resource->parent)
+-		devlink_resource_validate_children(resource->parent);
+-	return 0;
+-}
+-
+-static int
+-devlink_resource_size_params_put(struct devlink_resource *resource,
+-				 struct sk_buff *skb)
+-{
+-	struct devlink_resource_size_params *size_params;
+-
+-	size_params = &resource->size_params;
+-	if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
+-			      size_params->size_granularity, DEVLINK_ATTR_PAD) ||
+-	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
+-			      size_params->size_max, DEVLINK_ATTR_PAD) ||
+-	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
+-			      size_params->size_min, DEVLINK_ATTR_PAD) ||
+-	    nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
+-		return -EMSGSIZE;
+-	return 0;
+-}
+-
+-static int devlink_resource_occ_put(struct devlink_resource *resource,
+-				    struct sk_buff *skb)
+-{
+-	if (!resource->occ_get)
+-		return 0;
+-	return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
+-				 resource->occ_get(resource->occ_get_priv),
+-				 DEVLINK_ATTR_PAD);
+-}
+-
+-static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
+-				struct devlink_resource *resource)
+-{
+-	struct devlink_resource *child_resource;
+-	struct nlattr *child_resource_attr;
+-	struct nlattr *resource_attr;
+-
+-	resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
+-	if (!resource_attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
+-	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
+-			      DEVLINK_ATTR_PAD) ||
+-	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-	if (resource->size != resource->size_new)
+-		nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
+-				  resource->size_new, DEVLINK_ATTR_PAD);
+-	if (devlink_resource_occ_put(resource, skb))
+-		goto nla_put_failure;
+-	if (devlink_resource_size_params_put(resource, skb))
+-		goto nla_put_failure;
+-	if (list_empty(&resource->resource_list))
+-		goto out;
+-
+-	if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
+-		       resource->size_valid))
+-		goto nla_put_failure;
+-
+-	child_resource_attr = nla_nest_start_noflag(skb,
+-						    DEVLINK_ATTR_RESOURCE_LIST);
+-	if (!child_resource_attr)
+-		goto nla_put_failure;
+-
+-	list_for_each_entry(child_resource, &resource->resource_list, list) {
+-		if (devlink_resource_put(devlink, skb, child_resource))
+-			goto resource_put_failure;
+-	}
+-
+-	nla_nest_end(skb, child_resource_attr);
+-out:
+-	nla_nest_end(skb, resource_attr);
+-	return 0;
+-
+-resource_put_failure:
+-	nla_nest_cancel(skb, child_resource_attr);
+-nla_put_failure:
+-	nla_nest_cancel(skb, resource_attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_resource_fill(struct genl_info *info,
+-				 enum devlink_command cmd, int flags)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_resource *resource;
+-	struct nlattr *resources_attr;
+-	struct sk_buff *skb = NULL;
+-	struct nlmsghdr *nlh;
+-	bool incomplete;
+-	void *hdr;
+-	int i;
+-	int err;
+-
+-	resource = list_first_entry(&devlink->resource_list,
+-				    struct devlink_resource, list);
+-start_again:
+-	err = devlink_dpipe_send_and_alloc_skb(&skb, info);
+-	if (err)
+-		return err;
+-
+-	hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			  &devlink_nl_family, NLM_F_MULTI, cmd);
+-	if (!hdr) {
+-		nlmsg_free(skb);
+-		return -EMSGSIZE;
+-	}
+-
+-	if (devlink_nl_put_handle(skb, devlink))
+-		goto nla_put_failure;
+-
+-	resources_attr = nla_nest_start_noflag(skb,
+-					       DEVLINK_ATTR_RESOURCE_LIST);
+-	if (!resources_attr)
+-		goto nla_put_failure;
+-
+-	incomplete = false;
+-	i = 0;
+-	list_for_each_entry_from(resource, &devlink->resource_list, list) {
+-		err = devlink_resource_put(devlink, skb, resource);
+-		if (err) {
+-			if (!i)
+-				goto err_resource_put;
+-			incomplete = true;
+-			break;
+-		}
+-		i++;
+-	}
+-	nla_nest_end(skb, resources_attr);
+-	genlmsg_end(skb, hdr);
+-	if (incomplete)
+-		goto start_again;
+-send_done:
+-	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			NLMSG_DONE, 0, flags | NLM_F_MULTI);
+-	if (!nlh) {
+-		err = devlink_dpipe_send_and_alloc_skb(&skb, info);
+-		if (err)
+-			return err;
+-		goto send_done;
+-	}
+-	return genlmsg_reply(skb, info);
+-
+-nla_put_failure:
+-	err = -EMSGSIZE;
+-err_resource_put:
+-	nlmsg_free(skb);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-
+-	if (list_empty(&devlink->resource_list))
+-		return -EOPNOTSUPP;
+-
+-	return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
+-}
+-
+-static int
+-devlink_resources_validate(struct devlink *devlink,
+-			   struct devlink_resource *resource,
+-			   struct genl_info *info)
+-{
+-	struct list_head *resource_list;
+-	int err = 0;
+-
+-	if (resource)
+-		resource_list = &resource->resource_list;
+-	else
+-		resource_list = &devlink->resource_list;
+-
+-	list_for_each_entry(resource, resource_list, list) {
+-		if (!resource->size_valid)
+-			return -EINVAL;
+-		err = devlink_resources_validate(devlink, resource, info);
+-		if (err)
+-			return err;
+-	}
+-	return err;
+-}
+-
+-static struct net *devlink_netns_get(struct sk_buff *skb,
+-				     struct genl_info *info)
+-{
+-	struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
+-	struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
+-	struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
+-	struct net *net;
+-
+-	if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
+-		return ERR_PTR(-EINVAL);
+-	}
+-
+-	if (netns_pid_attr) {
+-		net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
+-	} else if (netns_fd_attr) {
+-		net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
+-	} else if (netns_id_attr) {
+-		net = get_net_ns_by_id(sock_net(skb->sk),
+-				       nla_get_u32(netns_id_attr));
+-		if (!net)
+-			net = ERR_PTR(-EINVAL);
+-	} else {
+-		WARN_ON(1);
+-		net = ERR_PTR(-EINVAL);
+-	}
+-	if (IS_ERR(net)) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
+-		return ERR_PTR(-EINVAL);
+-	}
+-	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
+-		put_net(net);
+-		return ERR_PTR(-EPERM);
+-	}
+-	return net;
+-}
+-
+-static void devlink_param_notify(struct devlink *devlink,
+-				 unsigned int port_index,
+-				 struct devlink_param_item *param_item,
+-				 enum devlink_command cmd);
+-
+-static void devlink_ns_change_notify(struct devlink *devlink,
+-				     struct net *dest_net, struct net *curr_net,
+-				     bool new)
+-{
+-	struct devlink_param_item *param_item;
+-	enum devlink_command cmd;
+-
+-	/* Userspace needs to be notified about devlink objects
+-	 * removed from original and entering new network namespace.
+-	 * The rest of the devlink objects are re-created during
+-	 * reload process so the notifications are generated separatelly.
+-	 */
+-
+-	if (!dest_net || net_eq(dest_net, curr_net))
+-		return;
+-
+-	if (new)
+-		devlink_notify(devlink, DEVLINK_CMD_NEW);
+-
+-	cmd = new ? DEVLINK_CMD_PARAM_NEW : DEVLINK_CMD_PARAM_DEL;
+-	list_for_each_entry(param_item, &devlink->param_list, list)
+-		devlink_param_notify(devlink, 0, param_item, cmd);
+-
+-	if (!new)
+-		devlink_notify(devlink, DEVLINK_CMD_DEL);
+-}
+-
+-static bool devlink_reload_supported(const struct devlink_ops *ops)
+-{
+-	return ops->reload_down && ops->reload_up;
+-}
+-
+-static void devlink_reload_failed_set(struct devlink *devlink,
+-				      bool reload_failed)
+-{
+-	if (devlink->reload_failed == reload_failed)
+-		return;
+-	devlink->reload_failed = reload_failed;
+-	devlink_notify(devlink, DEVLINK_CMD_NEW);
+-}
+-
+-bool devlink_is_reload_failed(const struct devlink *devlink)
+-{
+-	return devlink->reload_failed;
+-}
+-EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
+-
+-static void
+-__devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
+-			      enum devlink_reload_limit limit, u32 actions_performed)
+-{
+-	unsigned long actions = actions_performed;
+-	int stat_idx;
+-	int action;
+-
+-	for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
+-		stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
+-		reload_stats[stat_idx]++;
+-	}
+-	devlink_notify(devlink, DEVLINK_CMD_NEW);
+-}
+-
+-static void
+-devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
+-			    u32 actions_performed)
+-{
+-	__devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
+-				      actions_performed);
+-}
+-
+-/**
+- *	devlink_remote_reload_actions_performed - Update devlink on reload actions
+- *	  performed which are not a direct result of devlink reload call.
+- *
+- *	This should be called by a driver after performing reload actions in case it was not
+- *	a result of devlink reload call. For example fw_activate was performed as a result
+- *	of devlink reload triggered fw_activate on another host.
+- *	The motivation for this function is to keep data on reload actions performed on this
+- *	function whether it was done due to direct devlink reload call or not.
+- *
+- *	@devlink: devlink
+- *	@limit: reload limit
+- *	@actions_performed: bitmask of actions performed
+- */
+-void devlink_remote_reload_actions_performed(struct devlink *devlink,
+-					     enum devlink_reload_limit limit,
+-					     u32 actions_performed)
+-{
+-	if (WARN_ON(!actions_performed ||
+-		    actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
+-		    actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
+-		    limit > DEVLINK_RELOAD_LIMIT_MAX))
+-		return;
+-
+-	__devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
+-				      actions_performed);
+-}
+-EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
+-
+-static int devlink_reload(struct devlink *devlink, struct net *dest_net,
+-			  enum devlink_reload_action action, enum devlink_reload_limit limit,
+-			  u32 *actions_performed, struct netlink_ext_ack *extack)
+-{
+-	u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
+-	struct net *curr_net;
+-	int err;
+-
+-	memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
+-	       sizeof(remote_reload_stats));
+-
+-	curr_net = devlink_net(devlink);
+-	devlink_ns_change_notify(devlink, dest_net, curr_net, false);
+-	err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
+-	if (err)
+-		return err;
+-
+-	if (dest_net && !net_eq(dest_net, curr_net))
+-		write_pnet(&devlink->_net, dest_net);
+-
+-	err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
+-	devlink_reload_failed_set(devlink, !!err);
+-	if (err)
+-		return err;
+-
+-	devlink_ns_change_notify(devlink, dest_net, curr_net, true);
+-	WARN_ON(!(*actions_performed & BIT(action)));
+-	/* Catch driver on updating the remote action within devlink reload */
+-	WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
+-		       sizeof(remote_reload_stats)));
+-	devlink_reload_stats_update(devlink, limit, *actions_performed);
+-	return 0;
+-}
+-
+-static int
+-devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
+-					enum devlink_command cmd, struct genl_info *info)
+-{
+-	struct sk_buff *msg;
+-	void *hdr;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
+-	if (!hdr)
+-		goto free_msg;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
+-			       actions_performed))
+-		goto nla_put_failure;
+-	genlmsg_end(msg, hdr);
+-
+-	return genlmsg_reply(msg, info);
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-free_msg:
+-	nlmsg_free(msg);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	enum devlink_reload_action action;
+-	enum devlink_reload_limit limit;
+-	struct net *dest_net = NULL;
+-	u32 actions_performed;
+-	int err;
+-
+-	if (!(devlink->features & DEVLINK_F_RELOAD))
+-		return -EOPNOTSUPP;
+-
+-	err = devlink_resources_validate(devlink, NULL, info);
+-	if (err) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
+-		return err;
+-	}
+-
+-	if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
+-		action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
+-	else
+-		action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
+-
+-	if (!devlink_reload_action_is_supported(devlink, action)) {
+-		NL_SET_ERR_MSG_MOD(info->extack,
+-				   "Requested reload action is not supported by the driver");
+-		return -EOPNOTSUPP;
+-	}
+-
+-	limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
+-	if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
+-		struct nla_bitfield32 limits;
+-		u32 limits_selected;
+-
+-		limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
+-		limits_selected = limits.value & limits.selector;
+-		if (!limits_selected) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
+-			return -EINVAL;
+-		}
+-		for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
+-			if (limits_selected & BIT(limit))
+-				break;
+-		/* UAPI enables multiselection, but currently it is not used */
+-		if (limits_selected != BIT(limit)) {
+-			NL_SET_ERR_MSG_MOD(info->extack,
+-					   "Multiselection of limit is not supported");
+-			return -EOPNOTSUPP;
+-		}
+-		if (!devlink_reload_limit_is_supported(devlink, limit)) {
+-			NL_SET_ERR_MSG_MOD(info->extack,
+-					   "Requested limit is not supported by the driver");
+-			return -EOPNOTSUPP;
+-		}
+-		if (devlink_reload_combination_is_invalid(action, limit)) {
+-			NL_SET_ERR_MSG_MOD(info->extack,
+-					   "Requested limit is invalid for this action");
+-			return -EINVAL;
+-		}
+-	}
+-	if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
+-	    info->attrs[DEVLINK_ATTR_NETNS_FD] ||
+-	    info->attrs[DEVLINK_ATTR_NETNS_ID]) {
+-		dest_net = devlink_netns_get(skb, info);
+-		if (IS_ERR(dest_net))
+-			return PTR_ERR(dest_net);
+-	}
+-
+-	err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
+-
+-	if (dest_net)
+-		put_net(dest_net);
+-
+-	if (err)
+-		return err;
+-	/* For backward compatibility generate reply only if attributes used by user */
+-	if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
+-		return 0;
+-
+-	return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
+-						       DEVLINK_CMD_RELOAD, info);
+-}
+-
+-static int devlink_nl_flash_update_fill(struct sk_buff *msg,
+-					struct devlink *devlink,
+-					enum devlink_command cmd,
+-					struct devlink_flash_notify *params)
+-{
+-	void *hdr;
+-
+-	hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
+-		goto out;
+-
+-	if (params->status_msg &&
+-	    nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
+-			   params->status_msg))
+-		goto nla_put_failure;
+-	if (params->component &&
+-	    nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
+-			   params->component))
+-		goto nla_put_failure;
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
+-			      params->done, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
+-			      params->total, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
+-			      params->timeout, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-out:
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static void __devlink_flash_update_notify(struct devlink *devlink,
+-					  enum devlink_command cmd,
+-					  struct devlink_flash_notify *params)
+-{
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
+-		cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
+-		cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
+-
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
+-	if (err)
+-		goto out_free_msg;
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-	return;
+-
+-out_free_msg:
+-	nlmsg_free(msg);
+-}
+-
+-static void devlink_flash_update_begin_notify(struct devlink *devlink)
+-{
+-	struct devlink_flash_notify params = {};
+-
+-	__devlink_flash_update_notify(devlink,
+-				      DEVLINK_CMD_FLASH_UPDATE,
+-				      &params);
+-}
+-
+-static void devlink_flash_update_end_notify(struct devlink *devlink)
+-{
+-	struct devlink_flash_notify params = {};
+-
+-	__devlink_flash_update_notify(devlink,
+-				      DEVLINK_CMD_FLASH_UPDATE_END,
+-				      &params);
+-}
+-
+-void devlink_flash_update_status_notify(struct devlink *devlink,
+-					const char *status_msg,
+-					const char *component,
+-					unsigned long done,
+-					unsigned long total)
+-{
+-	struct devlink_flash_notify params = {
+-		.status_msg = status_msg,
+-		.component = component,
+-		.done = done,
+-		.total = total,
+-	};
+-
+-	__devlink_flash_update_notify(devlink,
+-				      DEVLINK_CMD_FLASH_UPDATE_STATUS,
+-				      &params);
+-}
+-EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
+-
+-void devlink_flash_update_timeout_notify(struct devlink *devlink,
+-					 const char *status_msg,
+-					 const char *component,
+-					 unsigned long timeout)
+-{
+-	struct devlink_flash_notify params = {
+-		.status_msg = status_msg,
+-		.component = component,
+-		.timeout = timeout,
+-	};
+-
+-	__devlink_flash_update_notify(devlink,
+-				      DEVLINK_CMD_FLASH_UPDATE_STATUS,
+-				      &params);
+-}
+-EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
+-
+-struct devlink_info_req {
+-	struct sk_buff *msg;
+-	void (*version_cb)(const char *version_name,
+-			   enum devlink_info_version_type version_type,
+-			   void *version_cb_priv);
+-	void *version_cb_priv;
+-};
+-
+-struct devlink_flash_component_lookup_ctx {
+-	const char *lookup_name;
+-	bool lookup_name_found;
+-};
+-
+-static void
+-devlink_flash_component_lookup_cb(const char *version_name,
+-				  enum devlink_info_version_type version_type,
+-				  void *version_cb_priv)
+-{
+-	struct devlink_flash_component_lookup_ctx *lookup_ctx = version_cb_priv;
+-
+-	if (version_type != DEVLINK_INFO_VERSION_TYPE_COMPONENT ||
+-	    lookup_ctx->lookup_name_found)
+-		return;
+-
+-	lookup_ctx->lookup_name_found =
+-		!strcmp(lookup_ctx->lookup_name, version_name);
+-}
+-
+-static int devlink_flash_component_get(struct devlink *devlink,
+-				       struct nlattr *nla_component,
+-				       const char **p_component,
+-				       struct netlink_ext_ack *extack)
+-{
+-	struct devlink_flash_component_lookup_ctx lookup_ctx = {};
+-	struct devlink_info_req req = {};
+-	const char *component;
+-	int ret;
+-
+-	if (!nla_component)
+-		return 0;
+-
+-	component = nla_data(nla_component);
+-
+-	if (!devlink->ops->info_get) {
+-		NL_SET_ERR_MSG_ATTR(extack, nla_component,
+-				    "component update is not supported by this device");
+-		return -EOPNOTSUPP;
+-	}
+-
+-	lookup_ctx.lookup_name = component;
+-	req.version_cb = devlink_flash_component_lookup_cb;
+-	req.version_cb_priv = &lookup_ctx;
+-
+-	ret = devlink->ops->info_get(devlink, &req, NULL);
+-	if (ret)
+-		return ret;
+-
+-	if (!lookup_ctx.lookup_name_found) {
+-		NL_SET_ERR_MSG_ATTR(extack, nla_component,
+-				    "selected component is not supported by this device");
+-		return -EINVAL;
+-	}
+-	*p_component = component;
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
+-				       struct genl_info *info)
+-{
+-	struct nlattr *nla_overwrite_mask, *nla_file_name;
+-	struct devlink_flash_update_params params = {};
+-	struct devlink *devlink = info->user_ptr[0];
+-	const char *file_name;
+-	u32 supported_params;
+-	int ret;
+-
+-	if (!devlink->ops->flash_update)
+-		return -EOPNOTSUPP;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME))
+-		return -EINVAL;
+-
+-	ret = devlink_flash_component_get(devlink,
+-					  info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT],
+-					  &params.component, info->extack);
+-	if (ret)
+-		return ret;
+-
+-	supported_params = devlink->ops->supported_flash_update_params;
+-
+-	nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
+-	if (nla_overwrite_mask) {
+-		struct nla_bitfield32 sections;
+-
+-		if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
+-			NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
+-					    "overwrite settings are not supported by this device");
+-			return -EOPNOTSUPP;
+-		}
+-		sections = nla_get_bitfield32(nla_overwrite_mask);
+-		params.overwrite_mask = sections.value & sections.selector;
+-	}
+-
+-	nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
+-	file_name = nla_data(nla_file_name);
+-	ret = request_firmware(&params.fw, file_name, devlink->dev);
+-	if (ret) {
+-		NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name, "failed to locate the requested firmware file");
+-		return ret;
+-	}
+-
+-	devlink_flash_update_begin_notify(devlink);
+-	ret = devlink->ops->flash_update(devlink, &params, info->extack);
+-	devlink_flash_update_end_notify(devlink);
+-
+-	release_firmware(params.fw);
+-
+-	return ret;
+-}
+-
+-static int
+-devlink_nl_selftests_fill(struct sk_buff *msg, struct devlink *devlink,
+-			  u32 portid, u32 seq, int flags,
+-			  struct netlink_ext_ack *extack)
+-{
+-	struct nlattr *selftests;
+-	void *hdr;
+-	int err;
+-	int i;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags,
+-			  DEVLINK_CMD_SELFTESTS_GET);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	err = -EMSGSIZE;
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto err_cancel_msg;
+-
+-	selftests = nla_nest_start(msg, DEVLINK_ATTR_SELFTESTS);
+-	if (!selftests)
+-		goto err_cancel_msg;
+-
+-	for (i = DEVLINK_ATTR_SELFTEST_ID_UNSPEC + 1;
+-	     i <= DEVLINK_ATTR_SELFTEST_ID_MAX; i++) {
+-		if (devlink->ops->selftest_check(devlink, i, extack)) {
+-			err = nla_put_flag(msg, i);
+-			if (err)
+-				goto err_cancel_msg;
+-		}
+-	}
+-
+-	nla_nest_end(msg, selftests);
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-err_cancel_msg:
+-	genlmsg_cancel(msg, hdr);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_selftests_get_doit(struct sk_buff *skb,
+-					     struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	if (!devlink->ops->selftest_check)
+-		return -EOPNOTSUPP;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_selftests_fill(msg, devlink, info->snd_portid,
+-					info->snd_seq, 0, info->extack);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_selftests_get_dumpit(struct sk_buff *msg,
+-					       struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		if (idx < start || !devlink->ops->selftest_check)
+-			goto inc;
+-
+-		devl_lock(devlink);
+-		err = devlink_nl_selftests_fill(msg, devlink,
+-						NETLINK_CB(cb->skb).portid,
+-						cb->nlh->nlmsg_seq, NLM_F_MULTI,
+-						cb->extack);
+-		devl_unlock(devlink);
+-		if (err) {
+-			devlink_put(devlink);
+-			break;
+-		}
+-inc:
+-		idx++;
+-		devlink_put(devlink);
+-	}
+-
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_selftest_result_put(struct sk_buff *skb, unsigned int id,
+-				       enum devlink_selftest_status test_status)
+-{
+-	struct nlattr *result_attr;
+-
+-	result_attr = nla_nest_start(skb, DEVLINK_ATTR_SELFTEST_RESULT);
+-	if (!result_attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_u32(skb, DEVLINK_ATTR_SELFTEST_RESULT_ID, id) ||
+-	    nla_put_u8(skb, DEVLINK_ATTR_SELFTEST_RESULT_STATUS,
+-		       test_status))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(skb, result_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(skb, result_attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_selftests_run(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct nlattr *tb[DEVLINK_ATTR_SELFTEST_ID_MAX + 1];
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct nlattr *attrs, *selftests;
+-	struct sk_buff *msg;
+-	void *hdr;
+-	int err;
+-	int i;
+-
+-	if (!devlink->ops->selftest_run || !devlink->ops->selftest_check)
+-		return -EOPNOTSUPP;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SELFTESTS))
+-		return -EINVAL;
+-
+-	attrs = info->attrs[DEVLINK_ATTR_SELFTESTS];
+-
+-	err = nla_parse_nested(tb, DEVLINK_ATTR_SELFTEST_ID_MAX, attrs,
+-			       devlink_selftest_nl_policy, info->extack);
+-	if (err < 0)
+-		return err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = -EMSGSIZE;
+-	hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
+-			  &devlink_nl_family, 0, DEVLINK_CMD_SELFTESTS_RUN);
+-	if (!hdr)
+-		goto free_msg;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto genlmsg_cancel;
+-
+-	selftests = nla_nest_start(msg, DEVLINK_ATTR_SELFTESTS);
+-	if (!selftests)
+-		goto genlmsg_cancel;
+-
+-	for (i = DEVLINK_ATTR_SELFTEST_ID_UNSPEC + 1;
+-	     i <= DEVLINK_ATTR_SELFTEST_ID_MAX; i++) {
+-		enum devlink_selftest_status test_status;
+-
+-		if (nla_get_flag(tb[i])) {
+-			if (!devlink->ops->selftest_check(devlink, i,
+-							  info->extack)) {
+-				if (devlink_selftest_result_put(msg, i,
+-								DEVLINK_SELFTEST_STATUS_SKIP))
+-					goto selftests_nest_cancel;
+-				continue;
+-			}
+-
+-			test_status = devlink->ops->selftest_run(devlink, i,
+-								 info->extack);
+-			if (devlink_selftest_result_put(msg, i, test_status))
+-				goto selftests_nest_cancel;
+-		}
+-	}
+-
+-	nla_nest_end(msg, selftests);
+-	genlmsg_end(msg, hdr);
+-	return genlmsg_reply(msg, info);
+-
+-selftests_nest_cancel:
+-	nla_nest_cancel(msg, selftests);
+-genlmsg_cancel:
+-	genlmsg_cancel(msg, hdr);
+-free_msg:
+-	nlmsg_free(msg);
+-	return err;
+-}
+-
+-static const struct devlink_param devlink_param_generic[] = {
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
+-		.name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
+-		.name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
+-		.name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
+-		.name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
+-		.name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
+-		.name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
+-		.name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
+-		.name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP,
+-		.name = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
+-		.name = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_TYPE,
+-	},
+-	{
+-		.id = DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
+-		.name = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME,
+-		.type = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE,
+-	},
+-};
+-
+-static int devlink_param_generic_verify(const struct devlink_param *param)
+-{
+-	/* verify it match generic parameter by id and name */
+-	if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
+-		return -EINVAL;
+-	if (strcmp(param->name, devlink_param_generic[param->id].name))
+-		return -ENOENT;
+-
+-	WARN_ON(param->type != devlink_param_generic[param->id].type);
+-
+-	return 0;
+-}
+-
+-static int devlink_param_driver_verify(const struct devlink_param *param)
+-{
+-	int i;
+-
+-	if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
+-		return -EINVAL;
+-	/* verify no such name in generic params */
+-	for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
+-		if (!strcmp(param->name, devlink_param_generic[i].name))
+-			return -EEXIST;
+-
+-	return 0;
+-}
+-
+-static struct devlink_param_item *
+-devlink_param_find_by_name(struct list_head *param_list,
+-			   const char *param_name)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	list_for_each_entry(param_item, param_list, list)
+-		if (!strcmp(param_item->param->name, param_name))
+-			return param_item;
+-	return NULL;
+-}
+-
+-static struct devlink_param_item *
+-devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	list_for_each_entry(param_item, param_list, list)
+-		if (param_item->param->id == param_id)
+-			return param_item;
+-	return NULL;
+-}
+-
+-static bool
+-devlink_param_cmode_is_supported(const struct devlink_param *param,
+-				 enum devlink_param_cmode cmode)
+-{
+-	return test_bit(cmode, &param->supported_cmodes);
+-}
+-
+-static int devlink_param_get(struct devlink *devlink,
+-			     const struct devlink_param *param,
+-			     struct devlink_param_gset_ctx *ctx)
+-{
+-	if (!param->get || devlink->reload_failed)
+-		return -EOPNOTSUPP;
+-	return param->get(devlink, param->id, ctx);
+-}
+-
+-static int devlink_param_set(struct devlink *devlink,
+-			     const struct devlink_param *param,
+-			     struct devlink_param_gset_ctx *ctx)
+-{
+-	if (!param->set || devlink->reload_failed)
+-		return -EOPNOTSUPP;
+-	return param->set(devlink, param->id, ctx);
+-}
+-
+-static int
+-devlink_param_type_to_nla_type(enum devlink_param_type param_type)
+-{
+-	switch (param_type) {
+-	case DEVLINK_PARAM_TYPE_U8:
+-		return NLA_U8;
+-	case DEVLINK_PARAM_TYPE_U16:
+-		return NLA_U16;
+-	case DEVLINK_PARAM_TYPE_U32:
+-		return NLA_U32;
+-	case DEVLINK_PARAM_TYPE_STRING:
+-		return NLA_STRING;
+-	case DEVLINK_PARAM_TYPE_BOOL:
+-		return NLA_FLAG;
+-	default:
+-		return -EINVAL;
+-	}
+-}
+-
+-static int
+-devlink_nl_param_value_fill_one(struct sk_buff *msg,
+-				enum devlink_param_type type,
+-				enum devlink_param_cmode cmode,
+-				union devlink_param_value val)
+-{
+-	struct nlattr *param_value_attr;
+-
+-	param_value_attr = nla_nest_start_noflag(msg,
+-						 DEVLINK_ATTR_PARAM_VALUE);
+-	if (!param_value_attr)
+-		goto nla_put_failure;
+-
+-	if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
+-		goto value_nest_cancel;
+-
+-	switch (type) {
+-	case DEVLINK_PARAM_TYPE_U8:
+-		if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
+-			goto value_nest_cancel;
+-		break;
+-	case DEVLINK_PARAM_TYPE_U16:
+-		if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
+-			goto value_nest_cancel;
+-		break;
+-	case DEVLINK_PARAM_TYPE_U32:
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
+-			goto value_nest_cancel;
+-		break;
+-	case DEVLINK_PARAM_TYPE_STRING:
+-		if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
+-				   val.vstr))
+-			goto value_nest_cancel;
+-		break;
+-	case DEVLINK_PARAM_TYPE_BOOL:
+-		if (val.vbool &&
+-		    nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
+-			goto value_nest_cancel;
+-		break;
+-	}
+-
+-	nla_nest_end(msg, param_value_attr);
+-	return 0;
+-
+-value_nest_cancel:
+-	nla_nest_cancel(msg, param_value_attr);
+-nla_put_failure:
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
+-				 unsigned int port_index,
+-				 struct devlink_param_item *param_item,
+-				 enum devlink_command cmd,
+-				 u32 portid, u32 seq, int flags)
+-{
+-	union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
+-	bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
+-	const struct devlink_param *param = param_item->param;
+-	struct devlink_param_gset_ctx ctx;
+-	struct nlattr *param_values_list;
+-	struct nlattr *param_attr;
+-	int nla_type;
+-	void *hdr;
+-	int err;
+-	int i;
+-
+-	/* Get value from driver part to driverinit configuration mode */
+-	for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
+-		if (!devlink_param_cmode_is_supported(param, i))
+-			continue;
+-		if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
+-			if (!param_item->driverinit_value_valid)
+-				return -EOPNOTSUPP;
+-			param_value[i] = param_item->driverinit_value;
+-		} else {
+-			ctx.cmode = i;
+-			err = devlink_param_get(devlink, param, &ctx);
+-			if (err)
+-				return err;
+-			param_value[i] = ctx.val;
+-		}
+-		param_value_set[i] = true;
+-	}
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto genlmsg_cancel;
+-
+-	if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
+-	    cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
+-	    cmd == DEVLINK_CMD_PORT_PARAM_DEL)
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
+-			goto genlmsg_cancel;
+-
+-	param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
+-	if (!param_attr)
+-		goto genlmsg_cancel;
+-	if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
+-		goto param_nest_cancel;
+-	if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
+-		goto param_nest_cancel;
+-
+-	nla_type = devlink_param_type_to_nla_type(param->type);
+-	if (nla_type < 0)
+-		goto param_nest_cancel;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
+-		goto param_nest_cancel;
+-
+-	param_values_list = nla_nest_start_noflag(msg,
+-						  DEVLINK_ATTR_PARAM_VALUES_LIST);
+-	if (!param_values_list)
+-		goto param_nest_cancel;
+-
+-	for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
+-		if (!param_value_set[i])
+-			continue;
+-		err = devlink_nl_param_value_fill_one(msg, param->type,
+-						      i, param_value[i]);
+-		if (err)
+-			goto values_list_nest_cancel;
+-	}
+-
+-	nla_nest_end(msg, param_values_list);
+-	nla_nest_end(msg, param_attr);
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-values_list_nest_cancel:
+-	nla_nest_end(msg, param_values_list);
+-param_nest_cancel:
+-	nla_nest_cancel(msg, param_attr);
+-genlmsg_cancel:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static void devlink_param_notify(struct devlink *devlink,
+-				 unsigned int port_index,
+-				 struct devlink_param_item *param_item,
+-				 enum devlink_command cmd)
+-{
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
+-		cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
+-		cmd != DEVLINK_CMD_PORT_PARAM_DEL);
+-	ASSERT_DEVLINK_REGISTERED(devlink);
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-	err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
+-				    0, 0, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
+-					   struct netlink_callback *cb)
+-{
+-	struct devlink_param_item *param_item;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(param_item, &devlink->param_list, list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_param_fill(msg, devlink, 0, param_item,
+-						    DEVLINK_CMD_PARAM_GET,
+-						    NETLINK_CB(cb->skb).portid,
+-						    cb->nlh->nlmsg_seq,
+-						    NLM_F_MULTI);
+-			if (err == -EOPNOTSUPP) {
+-				err = 0;
+-			} else if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int
+-devlink_param_type_get_from_info(struct genl_info *info,
+-				 enum devlink_param_type *param_type)
+-{
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_TYPE))
+-		return -EINVAL;
+-
+-	switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
+-	case NLA_U8:
+-		*param_type = DEVLINK_PARAM_TYPE_U8;
+-		break;
+-	case NLA_U16:
+-		*param_type = DEVLINK_PARAM_TYPE_U16;
+-		break;
+-	case NLA_U32:
+-		*param_type = DEVLINK_PARAM_TYPE_U32;
+-		break;
+-	case NLA_STRING:
+-		*param_type = DEVLINK_PARAM_TYPE_STRING;
+-		break;
+-	case NLA_FLAG:
+-		*param_type = DEVLINK_PARAM_TYPE_BOOL;
+-		break;
+-	default:
+-		return -EINVAL;
+-	}
+-
+-	return 0;
+-}
+-
+-static int
+-devlink_param_value_get_from_info(const struct devlink_param *param,
+-				  struct genl_info *info,
+-				  union devlink_param_value *value)
+-{
+-	struct nlattr *param_data;
+-	int len;
+-
+-	param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
+-
+-	if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
+-		return -EINVAL;
+-
+-	switch (param->type) {
+-	case DEVLINK_PARAM_TYPE_U8:
+-		if (nla_len(param_data) != sizeof(u8))
+-			return -EINVAL;
+-		value->vu8 = nla_get_u8(param_data);
+-		break;
+-	case DEVLINK_PARAM_TYPE_U16:
+-		if (nla_len(param_data) != sizeof(u16))
+-			return -EINVAL;
+-		value->vu16 = nla_get_u16(param_data);
+-		break;
+-	case DEVLINK_PARAM_TYPE_U32:
+-		if (nla_len(param_data) != sizeof(u32))
+-			return -EINVAL;
+-		value->vu32 = nla_get_u32(param_data);
+-		break;
+-	case DEVLINK_PARAM_TYPE_STRING:
+-		len = strnlen(nla_data(param_data), nla_len(param_data));
+-		if (len == nla_len(param_data) ||
+-		    len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
+-			return -EINVAL;
+-		strcpy(value->vstr, nla_data(param_data));
+-		break;
+-	case DEVLINK_PARAM_TYPE_BOOL:
+-		if (param_data && nla_len(param_data))
+-			return -EINVAL;
+-		value->vbool = nla_get_flag(param_data);
+-		break;
+-	}
+-	return 0;
+-}
+-
+-static struct devlink_param_item *
+-devlink_param_get_from_info(struct list_head *param_list,
+-			    struct genl_info *info)
+-{
+-	char *param_name;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_NAME))
+-		return NULL;
+-
+-	param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
+-	return devlink_param_find_by_name(param_list, param_name);
+-}
+-
+-static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
+-					 struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_param_item *param_item;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	param_item = devlink_param_get_from_info(&devlink->param_list, info);
+-	if (!param_item)
+-		return -EINVAL;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_param_fill(msg, devlink, 0, param_item,
+-				    DEVLINK_CMD_PARAM_GET,
+-				    info->snd_portid, info->snd_seq, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
+-					   unsigned int port_index,
+-					   struct list_head *param_list,
+-					   struct genl_info *info,
+-					   enum devlink_command cmd)
+-{
+-	enum devlink_param_type param_type;
+-	struct devlink_param_gset_ctx ctx;
+-	enum devlink_param_cmode cmode;
+-	struct devlink_param_item *param_item;
+-	const struct devlink_param *param;
+-	union devlink_param_value value;
+-	int err = 0;
+-
+-	param_item = devlink_param_get_from_info(param_list, info);
+-	if (!param_item)
+-		return -EINVAL;
+-	param = param_item->param;
+-	err = devlink_param_type_get_from_info(info, &param_type);
+-	if (err)
+-		return err;
+-	if (param_type != param->type)
+-		return -EINVAL;
+-	err = devlink_param_value_get_from_info(param, info, &value);
+-	if (err)
+-		return err;
+-	if (param->validate) {
+-		err = param->validate(devlink, param->id, value, info->extack);
+-		if (err)
+-			return err;
+-	}
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_VALUE_CMODE))
+-		return -EINVAL;
+-	cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
+-	if (!devlink_param_cmode_is_supported(param, cmode))
+-		return -EOPNOTSUPP;
+-
+-	if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
+-		if (param->type == DEVLINK_PARAM_TYPE_STRING)
+-			strcpy(param_item->driverinit_value.vstr, value.vstr);
+-		else
+-			param_item->driverinit_value = value;
+-		param_item->driverinit_value_valid = true;
+-	} else {
+-		if (!param->set)
+-			return -EOPNOTSUPP;
+-		ctx.val = value;
+-		ctx.cmode = cmode;
+-		err = devlink_param_set(devlink, param, &ctx);
+-		if (err)
+-			return err;
+-	}
+-
+-	devlink_param_notify(devlink, port_index, param_item, cmd);
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
+-					 struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-
+-	return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
+-					       info, DEVLINK_CMD_PARAM_NEW);
+-}
+-
+-static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
+-						struct netlink_callback *cb)
+-{
+-	NL_SET_ERR_MSG_MOD(cb->extack, "Port params are not supported");
+-	return msg->len;
+-}
+-
+-static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
+-					      struct genl_info *info)
+-{
+-	NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported");
+-	return -EINVAL;
+-}
+-
+-static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
+-					      struct genl_info *info)
+-{
+-	NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported");
+-	return -EINVAL;
+-}
+-
+-static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
+-					     struct devlink *devlink,
+-					     struct devlink_snapshot *snapshot)
+-{
+-	struct nlattr *snap_attr;
+-	int err;
+-
+-	snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
+-	if (!snap_attr)
+-		return -EINVAL;
+-
+-	err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, snap_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, snap_attr);
+-	return err;
+-}
+-
+-static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
+-					      struct devlink *devlink,
+-					      struct devlink_region *region)
+-{
+-	struct devlink_snapshot *snapshot;
+-	struct nlattr *snapshots_attr;
+-	int err;
+-
+-	snapshots_attr = nla_nest_start_noflag(msg,
+-					       DEVLINK_ATTR_REGION_SNAPSHOTS);
+-	if (!snapshots_attr)
+-		return -EINVAL;
+-
+-	list_for_each_entry(snapshot, &region->snapshot_list, list) {
+-		err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
+-		if (err)
+-			goto nla_put_failure;
+-	}
+-
+-	nla_nest_end(msg, snapshots_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, snapshots_attr);
+-	return err;
+-}
+-
+-static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
+-				  enum devlink_command cmd, u32 portid,
+-				  u32 seq, int flags,
+-				  struct devlink_region *region)
+-{
+-	void *hdr;
+-	int err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	err = devlink_nl_put_handle(msg, devlink);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	if (region->port) {
+-		err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
+-				  region->port->index);
+-		if (err)
+-			goto nla_put_failure;
+-	}
+-
+-	err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
+-				region->size,
+-				DEVLINK_ATTR_PAD);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	err = nla_put_u32(msg, DEVLINK_ATTR_REGION_MAX_SNAPSHOTS,
+-			  region->max_snapshots);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return err;
+-}
+-
+-static struct sk_buff *
+-devlink_nl_region_notify_build(struct devlink_region *region,
+-			       struct devlink_snapshot *snapshot,
+-			       enum devlink_command cmd, u32 portid, u32 seq)
+-{
+-	struct devlink *devlink = region->devlink;
+-	struct sk_buff *msg;
+-	void *hdr;
+-	int err;
+-
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return ERR_PTR(-ENOMEM);
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
+-	if (!hdr) {
+-		err = -EMSGSIZE;
+-		goto out_free_msg;
+-	}
+-
+-	err = devlink_nl_put_handle(msg, devlink);
+-	if (err)
+-		goto out_cancel_msg;
+-
+-	if (region->port) {
+-		err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
+-				  region->port->index);
+-		if (err)
+-			goto out_cancel_msg;
+-	}
+-
+-	err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
+-			     region->ops->name);
+-	if (err)
+-		goto out_cancel_msg;
+-
+-	if (snapshot) {
+-		err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
+-				  snapshot->id);
+-		if (err)
+-			goto out_cancel_msg;
+-	} else {
+-		err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
+-					region->size, DEVLINK_ATTR_PAD);
+-		if (err)
+-			goto out_cancel_msg;
+-	}
+-	genlmsg_end(msg, hdr);
+-
+-	return msg;
+-
+-out_cancel_msg:
+-	genlmsg_cancel(msg, hdr);
+-out_free_msg:
+-	nlmsg_free(msg);
+-	return ERR_PTR(err);
+-}
+-
+-static void devlink_nl_region_notify(struct devlink_region *region,
+-				     struct devlink_snapshot *snapshot,
+-				     enum devlink_command cmd)
+-{
+-	struct devlink *devlink = region->devlink;
+-	struct sk_buff *msg;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
+-	if (IS_ERR(msg))
+-		return;
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
+-				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-/**
+- * __devlink_snapshot_id_increment - Increment number of snapshots using an id
+- *	@devlink: devlink instance
+- *	@id: the snapshot id
+- *
+- *	Track when a new snapshot begins using an id. Load the count for the
+- *	given id from the snapshot xarray, increment it, and store it back.
+- *
+- *	Called when a new snapshot is created with the given id.
+- *
+- *	The id *must* have been previously allocated by
+- *	devlink_region_snapshot_id_get().
+- *
+- *	Returns 0 on success, or an error on failure.
+- */
+-static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
+-{
+-	unsigned long count;
+-	void *p;
+-	int err;
+-
+-	xa_lock(&devlink->snapshot_ids);
+-	p = xa_load(&devlink->snapshot_ids, id);
+-	if (WARN_ON(!p)) {
+-		err = -EINVAL;
+-		goto unlock;
+-	}
+-
+-	if (WARN_ON(!xa_is_value(p))) {
+-		err = -EINVAL;
+-		goto unlock;
+-	}
+-
+-	count = xa_to_value(p);
+-	count++;
+-
+-	err = xa_err(__xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
+-				GFP_ATOMIC));
+-unlock:
+-	xa_unlock(&devlink->snapshot_ids);
+-	return err;
+-}
+-
+-/**
+- * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
+- *	@devlink: devlink instance
+- *	@id: the snapshot id
+- *
+- *	Track when a snapshot is deleted and stops using an id. Load the count
+- *	for the given id from the snapshot xarray, decrement it, and store it
+- *	back.
+- *
+- *	If the count reaches zero, erase this id from the xarray, freeing it
+- *	up for future re-use by devlink_region_snapshot_id_get().
+- *
+- *	Called when a snapshot using the given id is deleted, and when the
+- *	initial allocator of the id is finished using it.
+- */
+-static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
+-{
+-	unsigned long count;
+-	void *p;
+-
+-	xa_lock(&devlink->snapshot_ids);
+-	p = xa_load(&devlink->snapshot_ids, id);
+-	if (WARN_ON(!p))
+-		goto unlock;
+-
+-	if (WARN_ON(!xa_is_value(p)))
+-		goto unlock;
+-
+-	count = xa_to_value(p);
+-
+-	if (count > 1) {
+-		count--;
+-		__xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
+-			   GFP_ATOMIC);
+-	} else {
+-		/* If this was the last user, we can erase this id */
+-		__xa_erase(&devlink->snapshot_ids, id);
+-	}
+-unlock:
+-	xa_unlock(&devlink->snapshot_ids);
+-}
+-
+-/**
+- *	__devlink_snapshot_id_insert - Insert a specific snapshot ID
+- *	@devlink: devlink instance
+- *	@id: the snapshot id
+- *
+- *	Mark the given snapshot id as used by inserting a zero value into the
+- *	snapshot xarray.
+- *
+- *	This must be called while holding the devlink instance lock. Unlike
+- *	devlink_snapshot_id_get, the initial reference count is zero, not one.
+- *	It is expected that the id will immediately be used before
+- *	releasing the devlink instance lock.
+- *
+- *	Returns zero on success, or an error code if the snapshot id could not
+- *	be inserted.
+- */
+-static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
+-{
+-	int err;
+-
+-	xa_lock(&devlink->snapshot_ids);
+-	if (xa_load(&devlink->snapshot_ids, id)) {
+-		xa_unlock(&devlink->snapshot_ids);
+-		return -EEXIST;
+-	}
+-	err = xa_err(__xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
+-				GFP_ATOMIC));
+-	xa_unlock(&devlink->snapshot_ids);
+-	return err;
+-}
+-
+-/**
+- *	__devlink_region_snapshot_id_get - get snapshot ID
+- *	@devlink: devlink instance
+- *	@id: storage to return snapshot id
+- *
+- *	Allocates a new snapshot id. Returns zero on success, or a negative
+- *	error on failure. Must be called while holding the devlink instance
+- *	lock.
+- *
+- *	Snapshot IDs are tracked using an xarray which stores the number of
+- *	users of the snapshot id.
+- *
+- *	Note that the caller of this function counts as a 'user', in order to
+- *	avoid race conditions. The caller must release its hold on the
+- *	snapshot by using devlink_region_snapshot_id_put.
+- */
+-static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
+-{
+-	return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
+-			xa_limit_32b, GFP_KERNEL);
+-}
+-
+-/**
+- *	__devlink_region_snapshot_create - create a new snapshot
+- *	This will add a new snapshot of a region. The snapshot
+- *	will be stored on the region struct and can be accessed
+- *	from devlink. This is useful for future analyses of snapshots.
+- *	Multiple snapshots can be created on a region.
+- *	The @snapshot_id should be obtained using the getter function.
+- *
+- *	Must be called only while holding the region snapshot lock.
+- *
+- *	@region: devlink region of the snapshot
+- *	@data: snapshot data
+- *	@snapshot_id: snapshot id to be created
+- */
+-static int
+-__devlink_region_snapshot_create(struct devlink_region *region,
+-				 u8 *data, u32 snapshot_id)
+-{
+-	struct devlink *devlink = region->devlink;
+-	struct devlink_snapshot *snapshot;
+-	int err;
+-
+-	lockdep_assert_held(&region->snapshot_lock);
+-
+-	/* check if region can hold one more snapshot */
+-	if (region->cur_snapshots == region->max_snapshots)
+-		return -ENOSPC;
+-
+-	if (devlink_region_snapshot_get_by_id(region, snapshot_id))
+-		return -EEXIST;
+-
+-	snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
+-	if (!snapshot)
+-		return -ENOMEM;
+-
+-	err = __devlink_snapshot_id_increment(devlink, snapshot_id);
+-	if (err)
+-		goto err_snapshot_id_increment;
+-
+-	snapshot->id = snapshot_id;
+-	snapshot->region = region;
+-	snapshot->data = data;
+-
+-	list_add_tail(&snapshot->list, &region->snapshot_list);
+-
+-	region->cur_snapshots++;
+-
+-	devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
+-	return 0;
+-
+-err_snapshot_id_increment:
+-	kfree(snapshot);
+-	return err;
+-}
+-
+-static void devlink_region_snapshot_del(struct devlink_region *region,
+-					struct devlink_snapshot *snapshot)
+-{
+-	struct devlink *devlink = region->devlink;
+-
+-	lockdep_assert_held(&region->snapshot_lock);
+-
+-	devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
+-	region->cur_snapshots--;
+-	list_del(&snapshot->list);
+-	region->ops->destructor(snapshot->data);
+-	__devlink_snapshot_id_decrement(devlink, snapshot->id);
+-	kfree(snapshot);
+-}
+-
+-static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
+-					  struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_port *port = NULL;
+-	struct devlink_region *region;
+-	const char *region_name;
+-	struct sk_buff *msg;
+-	unsigned int index;
+-	int err;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME))
+-		return -EINVAL;
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
+-		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
+-
+-		port = devlink_port_get_by_index(devlink, index);
+-		if (!port)
+-			return -ENODEV;
+-	}
+-
+-	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
+-	if (port)
+-		region = devlink_port_region_get_by_name(port, region_name);
+-	else
+-		region = devlink_region_get_by_name(devlink, region_name);
+-
+-	if (!region)
+-		return -EINVAL;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
+-				     info->snd_portid, info->snd_seq, 0,
+-				     region);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
+-						 struct netlink_callback *cb,
+-						 struct devlink_port *port,
+-						 int *idx,
+-						 int start)
+-{
+-	struct devlink_region *region;
+-	int err = 0;
+-
+-	list_for_each_entry(region, &port->region_list, list) {
+-		if (*idx < start) {
+-			(*idx)++;
+-			continue;
+-		}
+-		err = devlink_nl_region_fill(msg, port->devlink,
+-					     DEVLINK_CMD_REGION_GET,
+-					     NETLINK_CB(cb->skb).portid,
+-					     cb->nlh->nlmsg_seq,
+-					     NLM_F_MULTI, region);
+-		if (err)
+-			goto out;
+-		(*idx)++;
+-	}
+-
+-out:
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
+-						    struct netlink_callback *cb,
+-						    struct devlink *devlink,
+-						    int *idx,
+-						    int start)
+-{
+-	struct devlink_region *region;
+-	struct devlink_port *port;
+-	int err = 0;
+-
+-	devl_lock(devlink);
+-	list_for_each_entry(region, &devlink->region_list, list) {
+-		if (*idx < start) {
+-			(*idx)++;
+-			continue;
+-		}
+-		err = devlink_nl_region_fill(msg, devlink,
+-					     DEVLINK_CMD_REGION_GET,
+-					     NETLINK_CB(cb->skb).portid,
+-					     cb->nlh->nlmsg_seq,
+-					     NLM_F_MULTI, region);
+-		if (err)
+-			goto out;
+-		(*idx)++;
+-	}
+-
+-	list_for_each_entry(port, &devlink->port_list, list) {
+-		err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
+-							    start);
+-		if (err)
+-			goto out;
+-	}
+-
+-out:
+-	devl_unlock(devlink);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
+-					    struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
+-							       &idx, start);
+-		devlink_put(devlink);
+-		if (err)
+-			goto out;
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int devlink_nl_cmd_region_del(struct sk_buff *skb,
+-				     struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_snapshot *snapshot;
+-	struct devlink_port *port = NULL;
+-	struct devlink_region *region;
+-	const char *region_name;
+-	unsigned int index;
+-	u32 snapshot_id;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME) ||
+-	    GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_SNAPSHOT_ID))
+-		return -EINVAL;
+-
+-	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
+-	snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
+-		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
+-
+-		port = devlink_port_get_by_index(devlink, index);
+-		if (!port)
+-			return -ENODEV;
+-	}
+-
+-	if (port)
+-		region = devlink_port_region_get_by_name(port, region_name);
+-	else
+-		region = devlink_region_get_by_name(devlink, region_name);
+-
+-	if (!region)
+-		return -EINVAL;
+-
+-	mutex_lock(&region->snapshot_lock);
+-	snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
+-	if (!snapshot) {
+-		mutex_unlock(&region->snapshot_lock);
+-		return -EINVAL;
+-	}
+-
+-	devlink_region_snapshot_del(region, snapshot);
+-	mutex_unlock(&region->snapshot_lock);
+-	return 0;
+-}
+-
+-static int
+-devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_snapshot *snapshot;
+-	struct devlink_port *port = NULL;
+-	struct nlattr *snapshot_id_attr;
+-	struct devlink_region *region;
+-	const char *region_name;
+-	unsigned int index;
+-	u32 snapshot_id;
+-	u8 *data;
+-	int err;
+-
+-	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME)) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
+-		return -EINVAL;
+-	}
+-
+-	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
+-		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
+-
+-		port = devlink_port_get_by_index(devlink, index);
+-		if (!port)
+-			return -ENODEV;
+-	}
+-
+-	if (port)
+-		region = devlink_port_region_get_by_name(port, region_name);
+-	else
+-		region = devlink_region_get_by_name(devlink, region_name);
+-
+-	if (!region) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
+-		return -EINVAL;
+-	}
+-
+-	if (!region->ops->snapshot) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
+-		return -EOPNOTSUPP;
+-	}
+-
+-	mutex_lock(&region->snapshot_lock);
+-
+-	if (region->cur_snapshots == region->max_snapshots) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
+-		err = -ENOSPC;
+-		goto unlock;
+-	}
+-
+-	snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
+-	if (snapshot_id_attr) {
+-		snapshot_id = nla_get_u32(snapshot_id_attr);
+-
+-		if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
+-			err = -EEXIST;
+-			goto unlock;
+-		}
+-
+-		err = __devlink_snapshot_id_insert(devlink, snapshot_id);
+-		if (err)
+-			goto unlock;
+-	} else {
+-		err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
+-		if (err) {
+-			NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
+-			goto unlock;
+-		}
+-	}
+-
+-	if (port)
+-		err = region->port_ops->snapshot(port, region->port_ops,
+-						 info->extack, &data);
+-	else
+-		err = region->ops->snapshot(devlink, region->ops,
+-					    info->extack, &data);
+-	if (err)
+-		goto err_snapshot_capture;
+-
+-	err = __devlink_region_snapshot_create(region, data, snapshot_id);
+-	if (err)
+-		goto err_snapshot_create;
+-
+-	if (!snapshot_id_attr) {
+-		struct sk_buff *msg;
+-
+-		snapshot = devlink_region_snapshot_get_by_id(region,
+-							     snapshot_id);
+-		if (WARN_ON(!snapshot)) {
+-			err = -EINVAL;
+-			goto unlock;
+-		}
+-
+-		msg = devlink_nl_region_notify_build(region, snapshot,
+-						     DEVLINK_CMD_REGION_NEW,
+-						     info->snd_portid,
+-						     info->snd_seq);
+-		err = PTR_ERR_OR_ZERO(msg);
+-		if (err)
+-			goto err_notify;
+-
+-		err = genlmsg_reply(msg, info);
+-		if (err)
+-			goto err_notify;
+-	}
+-
+-	mutex_unlock(&region->snapshot_lock);
+-	return 0;
+-
+-err_snapshot_create:
+-	region->ops->destructor(data);
+-err_snapshot_capture:
+-	__devlink_snapshot_id_decrement(devlink, snapshot_id);
+-	mutex_unlock(&region->snapshot_lock);
+-	return err;
+-
+-err_notify:
+-	devlink_region_snapshot_del(region, snapshot);
+-unlock:
+-	mutex_unlock(&region->snapshot_lock);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
+-						 struct devlink *devlink,
+-						 u8 *chunk, u32 chunk_size,
+-						 u64 addr)
+-{
+-	struct nlattr *chunk_attr;
+-	int err;
+-
+-	chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
+-	if (!chunk_attr)
+-		return -EINVAL;
+-
+-	err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
+-				DEVLINK_ATTR_PAD);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, chunk_attr);
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, chunk_attr);
+-	return err;
+-}
+-
+-#define DEVLINK_REGION_READ_CHUNK_SIZE 256
+-
+-static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
+-						struct devlink *devlink,
+-						struct devlink_region *region,
+-						struct nlattr **attrs,
+-						u64 start_offset,
+-						u64 end_offset,
+-						u64 *new_offset)
+-{
+-	struct devlink_snapshot *snapshot;
+-	u64 curr_offset = start_offset;
+-	u32 snapshot_id;
+-	int err = 0;
+-
+-	*new_offset = start_offset;
+-
+-	snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
+-	snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
+-	if (!snapshot)
+-		return -EINVAL;
+-
+-	while (curr_offset < end_offset) {
+-		u32 data_size;
+-		u8 *data;
+-
+-		if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
+-			data_size = end_offset - curr_offset;
+-		else
+-			data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
+-
+-		data = &snapshot->data[curr_offset];
+-		err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
+-							    data, data_size,
+-							    curr_offset);
+-		if (err)
+-			break;
+-
+-		curr_offset += data_size;
+-	}
+-	*new_offset = curr_offset;
+-
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
+-					     struct netlink_callback *cb)
+-{
+-	const struct genl_dumpit_info *info = genl_dumpit_info(cb);
+-	u64 ret_offset, start_offset, end_offset = U64_MAX;
+-	struct nlattr **attrs = info->attrs;
+-	struct devlink_port *port = NULL;
+-	struct devlink_region *region;
+-	struct nlattr *chunks_attr;
+-	const char *region_name;
+-	struct devlink *devlink;
+-	unsigned int index;
+-	void *hdr;
+-	int err;
+-
+-	start_offset = *((u64 *)&cb->args[0]);
+-
+-	devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
+-	if (IS_ERR(devlink))
+-		return PTR_ERR(devlink);
+-
+-	devl_lock(devlink);
+-
+-	if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
+-	    !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
+-		err = -EINVAL;
+-		goto out_unlock;
+-	}
+-
+-	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
+-		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
+-
+-		port = devlink_port_get_by_index(devlink, index);
+-		if (!port) {
+-			err = -ENODEV;
+-			goto out_unlock;
+-		}
+-	}
+-
+-	region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
+-
+-	if (port)
+-		region = devlink_port_region_get_by_name(port, region_name);
+-	else
+-		region = devlink_region_get_by_name(devlink, region_name);
+-
+-	if (!region) {
+-		err = -EINVAL;
+-		goto out_unlock;
+-	}
+-
+-	if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
+-	    attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
+-		if (!start_offset)
+-			start_offset =
+-				nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
+-
+-		end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
+-		end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
+-	}
+-
+-	if (end_offset > region->size)
+-		end_offset = region->size;
+-
+-	/* return 0 if there is no further data to read */
+-	if (start_offset == end_offset) {
+-		err = 0;
+-		goto out_unlock;
+-	}
+-
+-	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+-			  &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
+-			  DEVLINK_CMD_REGION_READ);
+-	if (!hdr) {
+-		err = -EMSGSIZE;
+-		goto out_unlock;
+-	}
+-
+-	err = devlink_nl_put_handle(skb, devlink);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	if (region->port) {
+-		err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
+-				  region->port->index);
+-		if (err)
+-			goto nla_put_failure;
+-	}
+-
+-	err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
+-	if (!chunks_attr) {
+-		err = -EMSGSIZE;
+-		goto nla_put_failure;
+-	}
+-
+-	err = devlink_nl_region_read_snapshot_fill(skb, devlink,
+-						   region, attrs,
+-						   start_offset,
+-						   end_offset, &ret_offset);
+-
+-	if (err && err != -EMSGSIZE)
+-		goto nla_put_failure;
+-
+-	/* Check if there was any progress done to prevent infinite loop */
+-	if (ret_offset == start_offset) {
+-		err = -EINVAL;
+-		goto nla_put_failure;
+-	}
+-
+-	*((u64 *)&cb->args[0]) = ret_offset;
+-
+-	nla_nest_end(skb, chunks_attr);
+-	genlmsg_end(skb, hdr);
+-	devl_unlock(devlink);
+-	devlink_put(devlink);
+-	return skb->len;
+-
+-nla_put_failure:
+-	genlmsg_cancel(skb, hdr);
+-out_unlock:
+-	devl_unlock(devlink);
+-	devlink_put(devlink);
+-	return err;
+-}
+-
+-int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
+-{
+-	if (!req->msg)
+-		return 0;
+-	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
+-
+-int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
+-{
+-	if (!req->msg)
+-		return 0;
+-	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
+-
+-int devlink_info_board_serial_number_put(struct devlink_info_req *req,
+-					 const char *bsn)
+-{
+-	if (!req->msg)
+-		return 0;
+-	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
+-			      bsn);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
+-
+-static int devlink_info_version_put(struct devlink_info_req *req, int attr,
+-				    const char *version_name,
+-				    const char *version_value,
+-				    enum devlink_info_version_type version_type)
+-{
+-	struct nlattr *nest;
+-	int err;
+-
+-	if (req->version_cb)
+-		req->version_cb(version_name, version_type,
+-				req->version_cb_priv);
+-
+-	if (!req->msg)
+-		return 0;
+-
+-	nest = nla_nest_start_noflag(req->msg, attr);
+-	if (!nest)
+-		return -EMSGSIZE;
+-
+-	err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
+-			     version_name);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
+-			     version_value);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	nla_nest_end(req->msg, nest);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(req->msg, nest);
+-	return err;
+-}
+-
+-int devlink_info_version_fixed_put(struct devlink_info_req *req,
+-				   const char *version_name,
+-				   const char *version_value)
+-{
+-	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
+-					version_name, version_value,
+-					DEVLINK_INFO_VERSION_TYPE_NONE);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
+-
+-int devlink_info_version_stored_put(struct devlink_info_req *req,
+-				    const char *version_name,
+-				    const char *version_value)
+-{
+-	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
+-					version_name, version_value,
+-					DEVLINK_INFO_VERSION_TYPE_NONE);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
+-
+-int devlink_info_version_stored_put_ext(struct devlink_info_req *req,
+-					const char *version_name,
+-					const char *version_value,
+-					enum devlink_info_version_type version_type)
+-{
+-	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
+-					version_name, version_value,
+-					version_type);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_version_stored_put_ext);
+-
+-int devlink_info_version_running_put(struct devlink_info_req *req,
+-				     const char *version_name,
+-				     const char *version_value)
+-{
+-	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
+-					version_name, version_value,
+-					DEVLINK_INFO_VERSION_TYPE_NONE);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
+-
+-int devlink_info_version_running_put_ext(struct devlink_info_req *req,
+-					 const char *version_name,
+-					 const char *version_value,
+-					 enum devlink_info_version_type version_type)
+-{
+-	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
+-					version_name, version_value,
+-					version_type);
+-}
+-EXPORT_SYMBOL_GPL(devlink_info_version_running_put_ext);
+-
+-static int
+-devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
+-		     enum devlink_command cmd, u32 portid,
+-		     u32 seq, int flags, struct netlink_ext_ack *extack)
+-{
+-	struct devlink_info_req req = {};
+-	void *hdr;
+-	int err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	err = -EMSGSIZE;
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto err_cancel_msg;
+-
+-	req.msg = msg;
+-	err = devlink->ops->info_get(devlink, &req, extack);
+-	if (err)
+-		goto err_cancel_msg;
+-
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-err_cancel_msg:
+-	genlmsg_cancel(msg, hdr);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	if (!devlink->ops->info_get)
+-		return -EOPNOTSUPP;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
+-				   info->snd_portid, info->snd_seq, 0,
+-				   info->extack);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return err;
+-	}
+-
+-	return genlmsg_reply(msg, info);
+-}
+-
+-static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
+-					  struct netlink_callback *cb)
+-{
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err = 0;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		if (idx < start || !devlink->ops->info_get)
+-			goto inc;
+-
+-		devl_lock(devlink);
+-		err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
+-					   NETLINK_CB(cb->skb).portid,
+-					   cb->nlh->nlmsg_seq, NLM_F_MULTI,
+-					   cb->extack);
+-		devl_unlock(devlink);
+-		if (err == -EOPNOTSUPP)
+-			err = 0;
+-		else if (err) {
+-			devlink_put(devlink);
+-			break;
+-		}
+-inc:
+-		idx++;
+-		devlink_put(devlink);
+-	}
+-
+-	if (err != -EMSGSIZE)
+-		return err;
+-
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-struct devlink_fmsg_item {
+-	struct list_head list;
+-	int attrtype;
+-	u8 nla_type;
+-	u16 len;
+-	int value[];
+-};
+-
+-struct devlink_fmsg {
+-	struct list_head item_list;
+-	bool putting_binary; /* This flag forces enclosing of binary data
+-			      * in an array brackets. It forces using
+-			      * of designated API:
+-			      * devlink_fmsg_binary_pair_nest_start()
+-			      * devlink_fmsg_binary_pair_nest_end()
+-			      */
+-};
+-
+-static struct devlink_fmsg *devlink_fmsg_alloc(void)
+-{
+-	struct devlink_fmsg *fmsg;
+-
+-	fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
+-	if (!fmsg)
+-		return NULL;
+-
+-	INIT_LIST_HEAD(&fmsg->item_list);
+-
+-	return fmsg;
+-}
+-
+-static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
+-{
+-	struct devlink_fmsg_item *item, *tmp;
+-
+-	list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
+-		list_del(&item->list);
+-		kfree(item);
+-	}
+-	kfree(fmsg);
+-}
+-
+-static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
+-				    int attrtype)
+-{
+-	struct devlink_fmsg_item *item;
+-
+-	item = kzalloc(sizeof(*item), GFP_KERNEL);
+-	if (!item)
+-		return -ENOMEM;
+-
+-	item->attrtype = attrtype;
+-	list_add_tail(&item->list, &fmsg->item_list);
+-
+-	return 0;
+-}
+-
+-int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
+-
+-static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
+-}
+-
+-int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_nest_end(fmsg);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
+-
+-#define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
+-
+-static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
+-{
+-	struct devlink_fmsg_item *item;
+-
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
+-		return -EMSGSIZE;
+-
+-	item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
+-	if (!item)
+-		return -ENOMEM;
+-
+-	item->nla_type = NLA_NUL_STRING;
+-	item->len = strlen(name) + 1;
+-	item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
+-	memcpy(&item->value, name, item->len);
+-	list_add_tail(&item->list, &fmsg->item_list);
+-
+-	return 0;
+-}
+-
+-int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
+-{
+-	int err;
+-
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_put_name(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
+-
+-int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_nest_end(fmsg);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
+-
+-int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
+-				     const char *name)
+-{
+-	int err;
+-
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	err = devlink_fmsg_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
+-
+-int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
+-{
+-	int err;
+-
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	err = devlink_fmsg_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
+-
+-int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
+-					const char *name)
+-{
+-	int err;
+-
+-	err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	fmsg->putting_binary = true;
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
+-
+-int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
+-{
+-	if (!fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	fmsg->putting_binary = false;
+-	return devlink_fmsg_arr_pair_nest_end(fmsg);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
+-
+-static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
+-				  const void *value, u16 value_len,
+-				  u8 value_nla_type)
+-{
+-	struct devlink_fmsg_item *item;
+-
+-	if (value_len > DEVLINK_FMSG_MAX_SIZE)
+-		return -EMSGSIZE;
+-
+-	item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
+-	if (!item)
+-		return -ENOMEM;
+-
+-	item->nla_type = value_nla_type;
+-	item->len = value_len;
+-	item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
+-	memcpy(&item->value, value, item->len);
+-	list_add_tail(&item->list, &fmsg->item_list);
+-
+-	return 0;
+-}
+-
+-static int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
+-}
+-
+-static int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
+-}
+-
+-int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
+-
+-static int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
+-}
+-
+-int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
+-{
+-	if (fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
+-				      NLA_NUL_STRING);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
+-
+-int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
+-			    u16 value_len)
+-{
+-	if (!fmsg->putting_binary)
+-		return -EINVAL;
+-
+-	return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
+-
+-int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
+-			       bool value)
+-{
+-	int err;
+-
+-	err = devlink_fmsg_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_bool_put(fmsg, value);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_pair_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
+-
+-int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
+-			     u8 value)
+-{
+-	int err;
+-
+-	err = devlink_fmsg_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_u8_put(fmsg, value);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_pair_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
+-
+-int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
+-			      u32 value)
+-{
+-	int err;
+-
+-	err = devlink_fmsg_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_u32_put(fmsg, value);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_pair_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
+-
+-int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
+-			      u64 value)
+-{
+-	int err;
+-
+-	err = devlink_fmsg_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_u64_put(fmsg, value);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_pair_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
+-
+-int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
+-				 const char *value)
+-{
+-	int err;
+-
+-	err = devlink_fmsg_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_string_put(fmsg, value);
+-	if (err)
+-		return err;
+-
+-	err = devlink_fmsg_pair_nest_end(fmsg);
+-	if (err)
+-		return err;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
+-
+-int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
+-				 const void *value, u32 value_len)
+-{
+-	u32 data_size;
+-	int end_err;
+-	u32 offset;
+-	int err;
+-
+-	err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
+-	if (err)
+-		return err;
+-
+-	for (offset = 0; offset < value_len; offset += data_size) {
+-		data_size = value_len - offset;
+-		if (data_size > DEVLINK_FMSG_MAX_SIZE)
+-			data_size = DEVLINK_FMSG_MAX_SIZE;
+-		err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
+-		if (err)
+-			break;
+-		/* Exit from loop with a break (instead of
+-		 * return) to make sure putting_binary is turned off in
+-		 * devlink_fmsg_binary_pair_nest_end
+-		 */
+-	}
+-
+-	end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
+-	if (end_err)
+-		err = end_err;
+-
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
+-
+-static int
+-devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
+-{
+-	switch (msg->nla_type) {
+-	case NLA_FLAG:
+-	case NLA_U8:
+-	case NLA_U32:
+-	case NLA_U64:
+-	case NLA_NUL_STRING:
+-	case NLA_BINARY:
+-		return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
+-				  msg->nla_type);
+-	default:
+-		return -EINVAL;
+-	}
+-}
+-
+-static int
+-devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
+-{
+-	int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
+-	u8 tmp;
+-
+-	switch (msg->nla_type) {
+-	case NLA_FLAG:
+-		/* Always provide flag data, regardless of its value */
+-		tmp = *(bool *) msg->value;
+-
+-		return nla_put_u8(skb, attrtype, tmp);
+-	case NLA_U8:
+-		return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
+-	case NLA_U32:
+-		return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
+-	case NLA_U64:
+-		return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
+-					 DEVLINK_ATTR_PAD);
+-	case NLA_NUL_STRING:
+-		return nla_put_string(skb, attrtype, (char *) &msg->value);
+-	case NLA_BINARY:
+-		return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
+-	default:
+-		return -EINVAL;
+-	}
+-}
+-
+-static int
+-devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
+-			 int *start)
+-{
+-	struct devlink_fmsg_item *item;
+-	struct nlattr *fmsg_nlattr;
+-	int i = 0;
+-	int err;
+-
+-	fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
+-	if (!fmsg_nlattr)
+-		return -EMSGSIZE;
+-
+-	list_for_each_entry(item, &fmsg->item_list, list) {
+-		if (i < *start) {
+-			i++;
+-			continue;
+-		}
+-
+-		switch (item->attrtype) {
+-		case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
+-		case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
+-		case DEVLINK_ATTR_FMSG_ARR_NEST_START:
+-		case DEVLINK_ATTR_FMSG_NEST_END:
+-			err = nla_put_flag(skb, item->attrtype);
+-			break;
+-		case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
+-			err = devlink_fmsg_item_fill_type(item, skb);
+-			if (err)
+-				break;
+-			err = devlink_fmsg_item_fill_data(item, skb);
+-			break;
+-		case DEVLINK_ATTR_FMSG_OBJ_NAME:
+-			err = nla_put_string(skb, item->attrtype,
+-					     (char *) &item->value);
+-			break;
+-		default:
+-			err = -EINVAL;
+-			break;
+-		}
+-		if (!err)
+-			*start = ++i;
+-		else
+-			break;
+-	}
+-
+-	nla_nest_end(skb, fmsg_nlattr);
+-	return err;
+-}
+-
+-static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
+-			    struct genl_info *info,
+-			    enum devlink_command cmd, int flags)
+-{
+-	struct nlmsghdr *nlh;
+-	struct sk_buff *skb;
+-	bool last = false;
+-	int index = 0;
+-	void *hdr;
+-	int err;
+-
+-	while (!last) {
+-		int tmp_index = index;
+-
+-		skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-		if (!skb)
+-			return -ENOMEM;
+-
+-		hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
+-				  &devlink_nl_family, flags | NLM_F_MULTI, cmd);
+-		if (!hdr) {
+-			err = -EMSGSIZE;
+-			goto nla_put_failure;
+-		}
+-
+-		err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
+-		if (!err)
+-			last = true;
+-		else if (err != -EMSGSIZE || tmp_index == index)
+-			goto nla_put_failure;
+-
+-		genlmsg_end(skb, hdr);
+-		err = genlmsg_reply(skb, info);
+-		if (err)
+-			return err;
+-	}
+-
+-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!skb)
+-		return -ENOMEM;
+-	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
+-			NLMSG_DONE, 0, flags | NLM_F_MULTI);
+-	if (!nlh) {
+-		err = -EMSGSIZE;
+-		goto nla_put_failure;
+-	}
+-
+-	return genlmsg_reply(skb, info);
+-
+-nla_put_failure:
+-	nlmsg_free(skb);
+-	return err;
+-}
+-
+-static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
+-			       struct netlink_callback *cb,
+-			       enum devlink_command cmd)
+-{
+-	int index = cb->args[0];
+-	int tmp_index = index;
+-	void *hdr;
+-	int err;
+-
+-	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+-			  &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
+-	if (!hdr) {
+-		err = -EMSGSIZE;
+-		goto nla_put_failure;
+-	}
+-
+-	err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
+-	if ((err && err != -EMSGSIZE) || tmp_index == index)
+-		goto nla_put_failure;
+-
+-	cb->args[0] = index;
+-	genlmsg_end(skb, hdr);
+-	return skb->len;
+-
+-nla_put_failure:
+-	genlmsg_cancel(skb, hdr);
+-	return err;
+-}
+-
+-struct devlink_health_reporter {
+-	struct list_head list;
+-	void *priv;
+-	const struct devlink_health_reporter_ops *ops;
+-	struct devlink *devlink;
+-	struct devlink_port *devlink_port;
+-	struct devlink_fmsg *dump_fmsg;
+-	struct mutex dump_lock; /* lock parallel read/write from dump buffers */
+-	u64 graceful_period;
+-	bool auto_recover;
+-	bool auto_dump;
+-	u8 health_state;
+-	u64 dump_ts;
+-	u64 dump_real_ts;
+-	u64 error_count;
+-	u64 recovery_count;
+-	u64 last_recovery_ts;
+-	refcount_t refcount;
+-};
+-
+-void *
+-devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
+-{
+-	return reporter->priv;
+-}
+-EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
+-
+-static struct devlink_health_reporter *
+-__devlink_health_reporter_find_by_name(struct list_head *reporter_list,
+-				       struct mutex *list_lock,
+-				       const char *reporter_name)
+-{
+-	struct devlink_health_reporter *reporter;
+-
+-	lockdep_assert_held(list_lock);
+-	list_for_each_entry(reporter, reporter_list, list)
+-		if (!strcmp(reporter->ops->name, reporter_name))
+-			return reporter;
+-	return NULL;
+-}
+-
+-static struct devlink_health_reporter *
+-devlink_health_reporter_find_by_name(struct devlink *devlink,
+-				     const char *reporter_name)
+-{
+-	return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
+-						      &devlink->reporters_lock,
+-						      reporter_name);
+-}
+-
+-static struct devlink_health_reporter *
+-devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
+-					  const char *reporter_name)
+-{
+-	return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
+-						      &devlink_port->reporters_lock,
+-						      reporter_name);
+-}
+-
+-static struct devlink_health_reporter *
+-__devlink_health_reporter_create(struct devlink *devlink,
+-				 const struct devlink_health_reporter_ops *ops,
+-				 u64 graceful_period, void *priv)
+-{
+-	struct devlink_health_reporter *reporter;
+-
+-	if (WARN_ON(graceful_period && !ops->recover))
+-		return ERR_PTR(-EINVAL);
+-
+-	reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
+-	if (!reporter)
+-		return ERR_PTR(-ENOMEM);
+-
+-	reporter->priv = priv;
+-	reporter->ops = ops;
+-	reporter->devlink = devlink;
+-	reporter->graceful_period = graceful_period;
+-	reporter->auto_recover = !!ops->recover;
+-	reporter->auto_dump = !!ops->dump;
+-	mutex_init(&reporter->dump_lock);
+-	refcount_set(&reporter->refcount, 1);
+-	return reporter;
+-}
+-
+-/**
+- *	devlink_port_health_reporter_create - create devlink health reporter for
+- *	                                      specified port instance
+- *
+- *	@port: devlink_port which should contain the new reporter
+- *	@ops: ops
+- *	@graceful_period: to avoid recovery loops, in msecs
+- *	@priv: priv
+- */
+-struct devlink_health_reporter *
+-devlink_port_health_reporter_create(struct devlink_port *port,
+-				    const struct devlink_health_reporter_ops *ops,
+-				    u64 graceful_period, void *priv)
+-{
+-	struct devlink_health_reporter *reporter;
+-
+-	mutex_lock(&port->reporters_lock);
+-	if (__devlink_health_reporter_find_by_name(&port->reporter_list,
+-						   &port->reporters_lock, ops->name)) {
+-		reporter = ERR_PTR(-EEXIST);
+-		goto unlock;
+-	}
+-
+-	reporter = __devlink_health_reporter_create(port->devlink, ops,
+-						    graceful_period, priv);
+-	if (IS_ERR(reporter))
+-		goto unlock;
+-
+-	reporter->devlink_port = port;
+-	list_add_tail(&reporter->list, &port->reporter_list);
+-unlock:
+-	mutex_unlock(&port->reporters_lock);
+-	return reporter;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
+-
+-/**
+- *	devlink_health_reporter_create - create devlink health reporter
+- *
+- *	@devlink: devlink
+- *	@ops: ops
+- *	@graceful_period: to avoid recovery loops, in msecs
+- *	@priv: priv
+- */
+-struct devlink_health_reporter *
+-devlink_health_reporter_create(struct devlink *devlink,
+-			       const struct devlink_health_reporter_ops *ops,
+-			       u64 graceful_period, void *priv)
+-{
+-	struct devlink_health_reporter *reporter;
+-
+-	mutex_lock(&devlink->reporters_lock);
+-	if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
+-		reporter = ERR_PTR(-EEXIST);
+-		goto unlock;
+-	}
+-
+-	reporter = __devlink_health_reporter_create(devlink, ops,
+-						    graceful_period, priv);
+-	if (IS_ERR(reporter))
+-		goto unlock;
+-
+-	list_add_tail(&reporter->list, &devlink->reporter_list);
+-unlock:
+-	mutex_unlock(&devlink->reporters_lock);
+-	return reporter;
+-}
+-EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
+-
+-static void
+-devlink_health_reporter_free(struct devlink_health_reporter *reporter)
+-{
+-	mutex_destroy(&reporter->dump_lock);
+-	if (reporter->dump_fmsg)
+-		devlink_fmsg_free(reporter->dump_fmsg);
+-	kfree(reporter);
+-}
+-
+-static void
+-devlink_health_reporter_put(struct devlink_health_reporter *reporter)
+-{
+-	if (refcount_dec_and_test(&reporter->refcount))
+-		devlink_health_reporter_free(reporter);
+-}
+-
+-static void
+-__devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
+-{
+-	list_del(&reporter->list);
+-	devlink_health_reporter_put(reporter);
+-}
+-
+-/**
+- *	devlink_health_reporter_destroy - destroy devlink health reporter
+- *
+- *	@reporter: devlink health reporter to destroy
+- */
+-void
+-devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
+-{
+-	struct mutex *lock = &reporter->devlink->reporters_lock;
+-
+-	mutex_lock(lock);
+-	__devlink_health_reporter_destroy(reporter);
+-	mutex_unlock(lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
+-
+-/**
+- *	devlink_port_health_reporter_destroy - destroy devlink port health reporter
+- *
+- *	@reporter: devlink health reporter to destroy
+- */
+-void
+-devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
+-{
+-	struct mutex *lock = &reporter->devlink_port->reporters_lock;
+-
+-	mutex_lock(lock);
+-	__devlink_health_reporter_destroy(reporter);
+-	mutex_unlock(lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
+-
+-static int
+-devlink_nl_health_reporter_fill(struct sk_buff *msg,
+-				struct devlink_health_reporter *reporter,
+-				enum devlink_command cmd, u32 portid,
+-				u32 seq, int flags)
+-{
+-	struct devlink *devlink = reporter->devlink;
+-	struct nlattr *reporter_attr;
+-	void *hdr;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto genlmsg_cancel;
+-
+-	if (reporter->devlink_port) {
+-		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
+-			goto genlmsg_cancel;
+-	}
+-	reporter_attr = nla_nest_start_noflag(msg,
+-					      DEVLINK_ATTR_HEALTH_REPORTER);
+-	if (!reporter_attr)
+-		goto genlmsg_cancel;
+-	if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
+-			   reporter->ops->name))
+-		goto reporter_nest_cancel;
+-	if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
+-		       reporter->health_state))
+-		goto reporter_nest_cancel;
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
+-			      reporter->error_count, DEVLINK_ATTR_PAD))
+-		goto reporter_nest_cancel;
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
+-			      reporter->recovery_count, DEVLINK_ATTR_PAD))
+-		goto reporter_nest_cancel;
+-	if (reporter->ops->recover &&
+-	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
+-			      reporter->graceful_period,
+-			      DEVLINK_ATTR_PAD))
+-		goto reporter_nest_cancel;
+-	if (reporter->ops->recover &&
+-	    nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
+-		       reporter->auto_recover))
+-		goto reporter_nest_cancel;
+-	if (reporter->dump_fmsg &&
+-	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
+-			      jiffies_to_msecs(reporter->dump_ts),
+-			      DEVLINK_ATTR_PAD))
+-		goto reporter_nest_cancel;
+-	if (reporter->dump_fmsg &&
+-	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
+-			      reporter->dump_real_ts, DEVLINK_ATTR_PAD))
+-		goto reporter_nest_cancel;
+-	if (reporter->ops->dump &&
+-	    nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
+-		       reporter->auto_dump))
+-		goto reporter_nest_cancel;
+-
+-	nla_nest_end(msg, reporter_attr);
+-	genlmsg_end(msg, hdr);
+-	return 0;
+-
+-reporter_nest_cancel:
+-	nla_nest_end(msg, reporter_attr);
+-genlmsg_cancel:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static void devlink_recover_notify(struct devlink_health_reporter *reporter,
+-				   enum devlink_command cmd)
+-{
+-	struct devlink *devlink = reporter->devlink;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
+-	WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
+-				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-void
+-devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
+-{
+-	reporter->recovery_count++;
+-	reporter->last_recovery_ts = jiffies;
+-}
+-EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
+-
+-static int
+-devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
+-				void *priv_ctx, struct netlink_ext_ack *extack)
+-{
+-	int err;
+-
+-	if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
+-		return 0;
+-
+-	if (!reporter->ops->recover)
+-		return -EOPNOTSUPP;
+-
+-	err = reporter->ops->recover(reporter, priv_ctx, extack);
+-	if (err)
+-		return err;
+-
+-	devlink_health_reporter_recovery_done(reporter);
+-	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
+-	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
+-
+-	return 0;
+-}
+-
+-static void
+-devlink_health_dump_clear(struct devlink_health_reporter *reporter)
+-{
+-	if (!reporter->dump_fmsg)
+-		return;
+-	devlink_fmsg_free(reporter->dump_fmsg);
+-	reporter->dump_fmsg = NULL;
+-}
+-
+-static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
+-				  void *priv_ctx,
+-				  struct netlink_ext_ack *extack)
+-{
+-	int err;
+-
+-	if (!reporter->ops->dump)
+-		return 0;
+-
+-	if (reporter->dump_fmsg)
+-		return 0;
+-
+-	reporter->dump_fmsg = devlink_fmsg_alloc();
+-	if (!reporter->dump_fmsg) {
+-		err = -ENOMEM;
+-		return err;
+-	}
+-
+-	err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
+-	if (err)
+-		goto dump_err;
+-
+-	err = reporter->ops->dump(reporter, reporter->dump_fmsg,
+-				  priv_ctx, extack);
+-	if (err)
+-		goto dump_err;
+-
+-	err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
+-	if (err)
+-		goto dump_err;
+-
+-	reporter->dump_ts = jiffies;
+-	reporter->dump_real_ts = ktime_get_real_ns();
+-
+-	return 0;
+-
+-dump_err:
+-	devlink_health_dump_clear(reporter);
+-	return err;
+-}
+-
+-int devlink_health_report(struct devlink_health_reporter *reporter,
+-			  const char *msg, void *priv_ctx)
+-{
+-	enum devlink_health_reporter_state prev_health_state;
+-	struct devlink *devlink = reporter->devlink;
+-	unsigned long recover_ts_threshold;
+-	int ret;
+-
+-	/* write a log message of the current error */
+-	WARN_ON(!msg);
+-	trace_devlink_health_report(devlink, reporter->ops->name, msg);
+-	reporter->error_count++;
+-	prev_health_state = reporter->health_state;
+-	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
+-	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
+-
+-	/* abort if the previous error wasn't recovered */
+-	recover_ts_threshold = reporter->last_recovery_ts +
+-			       msecs_to_jiffies(reporter->graceful_period);
+-	if (reporter->auto_recover &&
+-	    (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
+-	     (reporter->last_recovery_ts && reporter->recovery_count &&
+-	      time_is_after_jiffies(recover_ts_threshold)))) {
+-		trace_devlink_health_recover_aborted(devlink,
+-						     reporter->ops->name,
+-						     reporter->health_state,
+-						     jiffies -
+-						     reporter->last_recovery_ts);
+-		return -ECANCELED;
+-	}
+-
+-	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
+-
+-	if (reporter->auto_dump) {
+-		mutex_lock(&reporter->dump_lock);
+-		/* store current dump of current error, for later analysis */
+-		devlink_health_do_dump(reporter, priv_ctx, NULL);
+-		mutex_unlock(&reporter->dump_lock);
+-	}
+-
+-	if (!reporter->auto_recover)
+-		return 0;
+-
+-	devl_lock(devlink);
+-	ret = devlink_health_reporter_recover(reporter, priv_ctx, NULL);
+-	devl_unlock(devlink);
+-
+-	return ret;
+-}
+-EXPORT_SYMBOL_GPL(devlink_health_report);
+-
+-static struct devlink_health_reporter *
+-devlink_health_reporter_get_from_attrs(struct devlink *devlink,
+-				       struct nlattr **attrs)
+-{
+-	struct devlink_health_reporter *reporter;
+-	struct devlink_port *devlink_port;
+-	char *reporter_name;
+-
+-	if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
+-		return NULL;
+-
+-	reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
+-	devlink_port = devlink_port_get_from_attrs(devlink, attrs);
+-	if (IS_ERR(devlink_port)) {
+-		mutex_lock(&devlink->reporters_lock);
+-		reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
+-		if (reporter)
+-			refcount_inc(&reporter->refcount);
+-		mutex_unlock(&devlink->reporters_lock);
+-	} else {
+-		mutex_lock(&devlink_port->reporters_lock);
+-		reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
+-		if (reporter)
+-			refcount_inc(&reporter->refcount);
+-		mutex_unlock(&devlink_port->reporters_lock);
+-	}
+-
+-	return reporter;
+-}
+-
+-static struct devlink_health_reporter *
+-devlink_health_reporter_get_from_info(struct devlink *devlink,
+-				      struct genl_info *info)
+-{
+-	return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
+-}
+-
+-static struct devlink_health_reporter *
+-devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
+-{
+-	const struct genl_dumpit_info *info = genl_dumpit_info(cb);
+-	struct devlink_health_reporter *reporter;
+-	struct nlattr **attrs = info->attrs;
+-	struct devlink *devlink;
+-
+-	devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
+-	if (IS_ERR(devlink))
+-		return NULL;
+-
+-	reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
+-	devlink_put(devlink);
+-	return reporter;
+-}
+-
+-void
+-devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
+-				     enum devlink_health_reporter_state state)
+-{
+-	if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
+-		    state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
+-		return;
+-
+-	if (reporter->health_state == state)
+-		return;
+-
+-	reporter->health_state = state;
+-	trace_devlink_health_reporter_state_update(reporter->devlink,
+-						   reporter->ops->name, state);
+-	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
+-}
+-EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
+-
+-static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
+-						   struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_health_reporter *reporter;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	reporter = devlink_health_reporter_get_from_info(devlink, info);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg) {
+-		err = -ENOMEM;
+-		goto out;
+-	}
+-
+-	err = devlink_nl_health_reporter_fill(msg, reporter,
+-					      DEVLINK_CMD_HEALTH_REPORTER_GET,
+-					      info->snd_portid, info->snd_seq,
+-					      0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		goto out;
+-	}
+-
+-	err = genlmsg_reply(msg, info);
+-out:
+-	devlink_health_reporter_put(reporter);
+-	return err;
+-}
+-
+-static int
+-devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
+-					  struct netlink_callback *cb)
+-{
+-	struct devlink_health_reporter *reporter;
+-	struct devlink_port *port;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		mutex_lock(&devlink->reporters_lock);
+-		list_for_each_entry(reporter, &devlink->reporter_list,
+-				    list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_health_reporter_fill(
+-				msg, reporter, DEVLINK_CMD_HEALTH_REPORTER_GET,
+-				NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+-				NLM_F_MULTI);
+-			if (err) {
+-				mutex_unlock(&devlink->reporters_lock);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		mutex_unlock(&devlink->reporters_lock);
+-		devlink_put(devlink);
+-	}
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(port, &devlink->port_list, list) {
+-			mutex_lock(&port->reporters_lock);
+-			list_for_each_entry(reporter, &port->reporter_list, list) {
+-				if (idx < start) {
+-					idx++;
+-					continue;
+-				}
+-				err = devlink_nl_health_reporter_fill(
+-					msg, reporter,
+-					DEVLINK_CMD_HEALTH_REPORTER_GET,
+-					NETLINK_CB(cb->skb).portid,
+-					cb->nlh->nlmsg_seq, NLM_F_MULTI);
+-				if (err) {
+-					mutex_unlock(&port->reporters_lock);
+-					devl_unlock(devlink);
+-					devlink_put(devlink);
+-					goto out;
+-				}
+-				idx++;
+-			}
+-			mutex_unlock(&port->reporters_lock);
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int
+-devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_health_reporter *reporter;
+-	int err;
+-
+-	reporter = devlink_health_reporter_get_from_info(devlink, info);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	if (!reporter->ops->recover &&
+-	    (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
+-	     info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
+-		err = -EOPNOTSUPP;
+-		goto out;
+-	}
+-	if (!reporter->ops->dump &&
+-	    info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
+-		err = -EOPNOTSUPP;
+-		goto out;
+-	}
+-
+-	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
+-		reporter->graceful_period =
+-			nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
+-
+-	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
+-		reporter->auto_recover =
+-			nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
+-
+-	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
+-		reporter->auto_dump =
+-		nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
+-
+-	devlink_health_reporter_put(reporter);
+-	return 0;
+-out:
+-	devlink_health_reporter_put(reporter);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
+-						       struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_health_reporter *reporter;
+-	int err;
+-
+-	reporter = devlink_health_reporter_get_from_info(devlink, info);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	err = devlink_health_reporter_recover(reporter, NULL, info->extack);
+-
+-	devlink_health_reporter_put(reporter);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
+-							struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_health_reporter *reporter;
+-	struct devlink_fmsg *fmsg;
+-	int err;
+-
+-	reporter = devlink_health_reporter_get_from_info(devlink, info);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	if (!reporter->ops->diagnose) {
+-		devlink_health_reporter_put(reporter);
+-		return -EOPNOTSUPP;
+-	}
+-
+-	fmsg = devlink_fmsg_alloc();
+-	if (!fmsg) {
+-		devlink_health_reporter_put(reporter);
+-		return -ENOMEM;
+-	}
+-
+-	err = devlink_fmsg_obj_nest_start(fmsg);
+-	if (err)
+-		goto out;
+-
+-	err = reporter->ops->diagnose(reporter, fmsg, info->extack);
+-	if (err)
+-		goto out;
+-
+-	err = devlink_fmsg_obj_nest_end(fmsg);
+-	if (err)
+-		goto out;
+-
+-	err = devlink_fmsg_snd(fmsg, info,
+-			       DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
+-
+-out:
+-	devlink_fmsg_free(fmsg);
+-	devlink_health_reporter_put(reporter);
+-	return err;
+-}
+-
+-static int
+-devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
+-					       struct netlink_callback *cb)
+-{
+-	struct devlink_health_reporter *reporter;
+-	u64 start = cb->args[0];
+-	int err;
+-
+-	reporter = devlink_health_reporter_get_from_cb(cb);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	if (!reporter->ops->dump) {
+-		err = -EOPNOTSUPP;
+-		goto out;
+-	}
+-	mutex_lock(&reporter->dump_lock);
+-	if (!start) {
+-		err = devlink_health_do_dump(reporter, NULL, cb->extack);
+-		if (err)
+-			goto unlock;
+-		cb->args[1] = reporter->dump_ts;
+-	}
+-	if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
+-		NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
+-		err = -EAGAIN;
+-		goto unlock;
+-	}
+-
+-	err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
+-				  DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
+-unlock:
+-	mutex_unlock(&reporter->dump_lock);
+-out:
+-	devlink_health_reporter_put(reporter);
+-	return err;
+-}
+-
+-static int
+-devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
+-					       struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_health_reporter *reporter;
+-
+-	reporter = devlink_health_reporter_get_from_info(devlink, info);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	if (!reporter->ops->dump) {
+-		devlink_health_reporter_put(reporter);
+-		return -EOPNOTSUPP;
+-	}
+-
+-	mutex_lock(&reporter->dump_lock);
+-	devlink_health_dump_clear(reporter);
+-	mutex_unlock(&reporter->dump_lock);
+-	devlink_health_reporter_put(reporter);
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
+-						    struct genl_info *info)
+-{
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_health_reporter *reporter;
+-	int err;
+-
+-	reporter = devlink_health_reporter_get_from_info(devlink, info);
+-	if (!reporter)
+-		return -EINVAL;
+-
+-	if (!reporter->ops->test) {
+-		devlink_health_reporter_put(reporter);
+-		return -EOPNOTSUPP;
+-	}
+-
+-	err = reporter->ops->test(reporter, info->extack);
+-
+-	devlink_health_reporter_put(reporter);
+-	return err;
+-}
+-
+-struct devlink_stats {
+-	u64_stats_t rx_bytes;
+-	u64_stats_t rx_packets;
+-	struct u64_stats_sync syncp;
+-};
+-
+-/**
+- * struct devlink_trap_policer_item - Packet trap policer attributes.
+- * @policer: Immutable packet trap policer attributes.
+- * @rate: Rate in packets / sec.
+- * @burst: Burst size in packets.
+- * @list: trap_policer_list member.
+- *
+- * Describes packet trap policer attributes. Created by devlink during trap
+- * policer registration.
+- */
+-struct devlink_trap_policer_item {
+-	const struct devlink_trap_policer *policer;
+-	u64 rate;
+-	u64 burst;
+-	struct list_head list;
+-};
+-
+-/**
+- * struct devlink_trap_group_item - Packet trap group attributes.
+- * @group: Immutable packet trap group attributes.
+- * @policer_item: Associated policer item. Can be NULL.
+- * @list: trap_group_list member.
+- * @stats: Trap group statistics.
+- *
+- * Describes packet trap group attributes. Created by devlink during trap
+- * group registration.
+- */
+-struct devlink_trap_group_item {
+-	const struct devlink_trap_group *group;
+-	struct devlink_trap_policer_item *policer_item;
+-	struct list_head list;
+-	struct devlink_stats __percpu *stats;
+-};
+-
+-/**
+- * struct devlink_trap_item - Packet trap attributes.
+- * @trap: Immutable packet trap attributes.
+- * @group_item: Associated group item.
+- * @list: trap_list member.
+- * @action: Trap action.
+- * @stats: Trap statistics.
+- * @priv: Driver private information.
+- *
+- * Describes both mutable and immutable packet trap attributes. Created by
+- * devlink during trap registration and used for all trap related operations.
+- */
+-struct devlink_trap_item {
+-	const struct devlink_trap *trap;
+-	struct devlink_trap_group_item *group_item;
+-	struct list_head list;
+-	enum devlink_trap_action action;
+-	struct devlink_stats __percpu *stats;
+-	void *priv;
+-};
+-
+-static struct devlink_trap_policer_item *
+-devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-
+-	list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
+-		if (policer_item->policer->id == id)
+-			return policer_item;
+-	}
+-
+-	return NULL;
+-}
+-
+-static struct devlink_trap_item *
+-devlink_trap_item_lookup(struct devlink *devlink, const char *name)
+-{
+-	struct devlink_trap_item *trap_item;
+-
+-	list_for_each_entry(trap_item, &devlink->trap_list, list) {
+-		if (!strcmp(trap_item->trap->name, name))
+-			return trap_item;
+-	}
+-
+-	return NULL;
+-}
+-
+-static struct devlink_trap_item *
+-devlink_trap_item_get_from_info(struct devlink *devlink,
+-				struct genl_info *info)
+-{
+-	struct nlattr *attr;
+-
+-	if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
+-		return NULL;
+-	attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
+-
+-	return devlink_trap_item_lookup(devlink, nla_data(attr));
+-}
+-
+-static int
+-devlink_trap_action_get_from_info(struct genl_info *info,
+-				  enum devlink_trap_action *p_trap_action)
+-{
+-	u8 val;
+-
+-	val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
+-	switch (val) {
+-	case DEVLINK_TRAP_ACTION_DROP:
+-	case DEVLINK_TRAP_ACTION_TRAP:
+-	case DEVLINK_TRAP_ACTION_MIRROR:
+-		*p_trap_action = val;
+-		break;
+-	default:
+-		return -EINVAL;
+-	}
+-
+-	return 0;
+-}
+-
+-static int devlink_trap_metadata_put(struct sk_buff *msg,
+-				     const struct devlink_trap *trap)
+-{
+-	struct nlattr *attr;
+-
+-	attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
+-	if (!attr)
+-		return -EMSGSIZE;
+-
+-	if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
+-	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
+-		goto nla_put_failure;
+-	if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
+-	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, attr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, attr);
+-	return -EMSGSIZE;
+-}
+-
+-static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
+-				    struct devlink_stats *stats)
+-{
+-	int i;
+-
+-	memset(stats, 0, sizeof(*stats));
+-	for_each_possible_cpu(i) {
+-		struct devlink_stats *cpu_stats;
+-		u64 rx_packets, rx_bytes;
+-		unsigned int start;
+-
+-		cpu_stats = per_cpu_ptr(trap_stats, i);
+-		do {
+-			start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
+-			rx_packets = u64_stats_read(&cpu_stats->rx_packets);
+-			rx_bytes = u64_stats_read(&cpu_stats->rx_bytes);
+-		} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
+-
+-		u64_stats_add(&stats->rx_packets, rx_packets);
+-		u64_stats_add(&stats->rx_bytes, rx_bytes);
+-	}
+-}
+-
+-static int
+-devlink_trap_group_stats_put(struct sk_buff *msg,
+-			     struct devlink_stats __percpu *trap_stats)
+-{
+-	struct devlink_stats stats;
+-	struct nlattr *attr;
+-
+-	devlink_trap_stats_read(trap_stats, &stats);
+-
+-	attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
+-	if (!attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
+-			      u64_stats_read(&stats.rx_packets),
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
+-			      u64_stats_read(&stats.rx_bytes),
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, attr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink,
+-				  const struct devlink_trap_item *trap_item)
+-{
+-	struct devlink_stats stats;
+-	struct nlattr *attr;
+-	u64 drops = 0;
+-	int err;
+-
+-	if (devlink->ops->trap_drop_counter_get) {
+-		err = devlink->ops->trap_drop_counter_get(devlink,
+-							  trap_item->trap,
+-							  &drops);
+-		if (err)
+-			return err;
+-	}
+-
+-	devlink_trap_stats_read(trap_item->stats, &stats);
+-
+-	attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
+-	if (!attr)
+-		return -EMSGSIZE;
+-
+-	if (devlink->ops->trap_drop_counter_get &&
+-	    nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
+-			      u64_stats_read(&stats.rx_packets),
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
+-			      u64_stats_read(&stats.rx_bytes),
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, attr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
+-				const struct devlink_trap_item *trap_item,
+-				enum devlink_command cmd, u32 portid, u32 seq,
+-				int flags)
+-{
+-	struct devlink_trap_group_item *group_item = trap_item->group_item;
+-	void *hdr;
+-	int err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
+-			   group_item->group->name))
+-		goto nla_put_failure;
+-
+-	if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
+-		goto nla_put_failure;
+-
+-	if (trap_item->trap->generic &&
+-	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
+-		goto nla_put_failure;
+-
+-	err = devlink_trap_metadata_put(msg, trap_item->trap);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	err = devlink_trap_stats_put(msg, devlink, trap_item);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_trap_item *trap_item;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	if (list_empty(&devlink->trap_list))
+-		return -EOPNOTSUPP;
+-
+-	trap_item = devlink_trap_item_get_from_info(devlink, info);
+-	if (!trap_item) {
+-		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
+-		return -ENOENT;
+-	}
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_trap_fill(msg, devlink, trap_item,
+-				   DEVLINK_CMD_TRAP_NEW, info->snd_portid,
+-				   info->snd_seq, 0);
+-	if (err)
+-		goto err_trap_fill;
+-
+-	return genlmsg_reply(msg, info);
+-
+-err_trap_fill:
+-	nlmsg_free(msg);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
+-					  struct netlink_callback *cb)
+-{
+-	struct devlink_trap_item *trap_item;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(trap_item, &devlink->trap_list, list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_trap_fill(msg, devlink, trap_item,
+-						   DEVLINK_CMD_TRAP_NEW,
+-						   NETLINK_CB(cb->skb).portid,
+-						   cb->nlh->nlmsg_seq,
+-						   NLM_F_MULTI);
+-			if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int __devlink_trap_action_set(struct devlink *devlink,
+-				     struct devlink_trap_item *trap_item,
+-				     enum devlink_trap_action trap_action,
+-				     struct netlink_ext_ack *extack)
+-{
+-	int err;
+-
+-	if (trap_item->action != trap_action &&
+-	    trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
+-		NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
+-		return 0;
+-	}
+-
+-	err = devlink->ops->trap_action_set(devlink, trap_item->trap,
+-					    trap_action, extack);
+-	if (err)
+-		return err;
+-
+-	trap_item->action = trap_action;
+-
+-	return 0;
+-}
+-
+-static int devlink_trap_action_set(struct devlink *devlink,
+-				   struct devlink_trap_item *trap_item,
+-				   struct genl_info *info)
+-{
+-	enum devlink_trap_action trap_action;
+-	int err;
+-
+-	if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
+-		return 0;
+-
+-	err = devlink_trap_action_get_from_info(info, &trap_action);
+-	if (err) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
+-		return -EINVAL;
+-	}
+-
+-	return __devlink_trap_action_set(devlink, trap_item, trap_action,
+-					 info->extack);
+-}
+-
+-static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
+-					struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_trap_item *trap_item;
+-
+-	if (list_empty(&devlink->trap_list))
+-		return -EOPNOTSUPP;
+-
+-	trap_item = devlink_trap_item_get_from_info(devlink, info);
+-	if (!trap_item) {
+-		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
+-		return -ENOENT;
+-	}
+-
+-	return devlink_trap_action_set(devlink, trap_item, info);
+-}
+-
+-static struct devlink_trap_group_item *
+-devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
+-{
+-	struct devlink_trap_group_item *group_item;
+-
+-	list_for_each_entry(group_item, &devlink->trap_group_list, list) {
+-		if (!strcmp(group_item->group->name, name))
+-			return group_item;
+-	}
+-
+-	return NULL;
+-}
+-
+-static struct devlink_trap_group_item *
+-devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
+-{
+-	struct devlink_trap_group_item *group_item;
+-
+-	list_for_each_entry(group_item, &devlink->trap_group_list, list) {
+-		if (group_item->group->id == id)
+-			return group_item;
+-	}
+-
+-	return NULL;
+-}
+-
+-static struct devlink_trap_group_item *
+-devlink_trap_group_item_get_from_info(struct devlink *devlink,
+-				      struct genl_info *info)
+-{
+-	char *name;
+-
+-	if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
+-		return NULL;
+-	name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
+-
+-	return devlink_trap_group_item_lookup(devlink, name);
+-}
+-
+-static int
+-devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
+-			   const struct devlink_trap_group_item *group_item,
+-			   enum devlink_command cmd, u32 portid, u32 seq,
+-			   int flags)
+-{
+-	void *hdr;
+-	int err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
+-			   group_item->group->name))
+-		goto nla_put_failure;
+-
+-	if (group_item->group->generic &&
+-	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
+-		goto nla_put_failure;
+-
+-	if (group_item->policer_item &&
+-	    nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
+-			group_item->policer_item->policer->id))
+-		goto nla_put_failure;
+-
+-	err = devlink_trap_group_stats_put(msg, group_item->stats);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
+-					      struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_trap_group_item *group_item;
+-	struct sk_buff *msg;
+-	int err;
+-
+-	if (list_empty(&devlink->trap_group_list))
+-		return -EOPNOTSUPP;
+-
+-	group_item = devlink_trap_group_item_get_from_info(devlink, info);
+-	if (!group_item) {
+-		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
+-		return -ENOENT;
+-	}
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_trap_group_fill(msg, devlink, group_item,
+-					 DEVLINK_CMD_TRAP_GROUP_NEW,
+-					 info->snd_portid, info->snd_seq, 0);
+-	if (err)
+-		goto err_trap_group_fill;
+-
+-	return genlmsg_reply(msg, info);
+-
+-err_trap_group_fill:
+-	nlmsg_free(msg);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
+-						struct netlink_callback *cb)
+-{
+-	enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
+-	struct devlink_trap_group_item *group_item;
+-	u32 portid = NETLINK_CB(cb->skb).portid;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(group_item, &devlink->trap_group_list,
+-				    list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_trap_group_fill(msg, devlink,
+-							 group_item, cmd,
+-							 portid,
+-							 cb->nlh->nlmsg_seq,
+-							 NLM_F_MULTI);
+-			if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int
+-__devlink_trap_group_action_set(struct devlink *devlink,
+-				struct devlink_trap_group_item *group_item,
+-				enum devlink_trap_action trap_action,
+-				struct netlink_ext_ack *extack)
+-{
+-	const char *group_name = group_item->group->name;
+-	struct devlink_trap_item *trap_item;
+-	int err;
+-
+-	if (devlink->ops->trap_group_action_set) {
+-		err = devlink->ops->trap_group_action_set(devlink, group_item->group,
+-							  trap_action, extack);
+-		if (err)
+-			return err;
+-
+-		list_for_each_entry(trap_item, &devlink->trap_list, list) {
+-			if (strcmp(trap_item->group_item->group->name, group_name))
+-				continue;
+-			if (trap_item->action != trap_action &&
+-			    trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
+-				continue;
+-			trap_item->action = trap_action;
+-		}
+-
+-		return 0;
+-	}
+-
+-	list_for_each_entry(trap_item, &devlink->trap_list, list) {
+-		if (strcmp(trap_item->group_item->group->name, group_name))
+-			continue;
+-		err = __devlink_trap_action_set(devlink, trap_item,
+-						trap_action, extack);
+-		if (err)
+-			return err;
+-	}
+-
+-	return 0;
+-}
+-
+-static int
+-devlink_trap_group_action_set(struct devlink *devlink,
+-			      struct devlink_trap_group_item *group_item,
+-			      struct genl_info *info, bool *p_modified)
+-{
+-	enum devlink_trap_action trap_action;
+-	int err;
+-
+-	if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
+-		return 0;
+-
+-	err = devlink_trap_action_get_from_info(info, &trap_action);
+-	if (err) {
+-		NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
+-		return -EINVAL;
+-	}
+-
+-	err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
+-					      info->extack);
+-	if (err)
+-		return err;
+-
+-	*p_modified = true;
+-
+-	return 0;
+-}
+-
+-static int devlink_trap_group_set(struct devlink *devlink,
+-				  struct devlink_trap_group_item *group_item,
+-				  struct genl_info *info)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-	struct netlink_ext_ack *extack = info->extack;
+-	const struct devlink_trap_policer *policer;
+-	struct nlattr **attrs = info->attrs;
+-	int err;
+-
+-	if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
+-		return 0;
+-
+-	if (!devlink->ops->trap_group_set)
+-		return -EOPNOTSUPP;
+-
+-	policer_item = group_item->policer_item;
+-	if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
+-		u32 policer_id;
+-
+-		policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
+-		policer_item = devlink_trap_policer_item_lookup(devlink,
+-								policer_id);
+-		if (policer_id && !policer_item) {
+-			NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
+-			return -ENOENT;
+-		}
+-	}
+-	policer = policer_item ? policer_item->policer : NULL;
+-
+-	err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
+-					   extack);
+-	if (err)
+-		return err;
+-
+-	group_item->policer_item = policer_item;
+-
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
+-					      struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct devlink_trap_group_item *group_item;
+-	bool modified = false;
+-	int err;
+-
+-	if (list_empty(&devlink->trap_group_list))
+-		return -EOPNOTSUPP;
+-
+-	group_item = devlink_trap_group_item_get_from_info(devlink, info);
+-	if (!group_item) {
+-		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
+-		return -ENOENT;
+-	}
+-
+-	err = devlink_trap_group_action_set(devlink, group_item, info,
+-					    &modified);
+-	if (err)
+-		return err;
+-
+-	err = devlink_trap_group_set(devlink, group_item, info);
+-	if (err)
+-		goto err_trap_group_set;
+-
+-	return 0;
+-
+-err_trap_group_set:
+-	if (modified)
+-		NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
+-	return err;
+-}
+-
+-static struct devlink_trap_policer_item *
+-devlink_trap_policer_item_get_from_info(struct devlink *devlink,
+-					struct genl_info *info)
+-{
+-	u32 id;
+-
+-	if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
+-		return NULL;
+-	id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
+-
+-	return devlink_trap_policer_item_lookup(devlink, id);
+-}
+-
+-static int
+-devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
+-			       const struct devlink_trap_policer *policer)
+-{
+-	struct nlattr *attr;
+-	u64 drops;
+-	int err;
+-
+-	if (!devlink->ops->trap_policer_counter_get)
+-		return 0;
+-
+-	err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
+-	if (err)
+-		return err;
+-
+-	attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
+-	if (!attr)
+-		return -EMSGSIZE;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
+-			      DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	nla_nest_end(msg, attr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	nla_nest_cancel(msg, attr);
+-	return -EMSGSIZE;
+-}
+-
+-static int
+-devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
+-			     const struct devlink_trap_policer_item *policer_item,
+-			     enum devlink_command cmd, u32 portid, u32 seq,
+-			     int flags)
+-{
+-	void *hdr;
+-	int err;
+-
+-	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
+-	if (!hdr)
+-		return -EMSGSIZE;
+-
+-	if (devlink_nl_put_handle(msg, devlink))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
+-			policer_item->policer->id))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
+-			      policer_item->rate, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
+-			      policer_item->burst, DEVLINK_ATTR_PAD))
+-		goto nla_put_failure;
+-
+-	err = devlink_trap_policer_stats_put(msg, devlink,
+-					     policer_item->policer);
+-	if (err)
+-		goto nla_put_failure;
+-
+-	genlmsg_end(msg, hdr);
+-
+-	return 0;
+-
+-nla_put_failure:
+-	genlmsg_cancel(msg, hdr);
+-	return -EMSGSIZE;
+-}
+-
+-static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
+-						struct genl_info *info)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-	struct sk_buff *msg;
+-	int err;
+-
+-	if (list_empty(&devlink->trap_policer_list))
+-		return -EOPNOTSUPP;
+-
+-	policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
+-	if (!policer_item) {
+-		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
+-		return -ENOENT;
+-	}
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return -ENOMEM;
+-
+-	err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
+-					   DEVLINK_CMD_TRAP_POLICER_NEW,
+-					   info->snd_portid, info->snd_seq, 0);
+-	if (err)
+-		goto err_trap_policer_fill;
+-
+-	return genlmsg_reply(msg, info);
+-
+-err_trap_policer_fill:
+-	nlmsg_free(msg);
+-	return err;
+-}
+-
+-static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
+-						  struct netlink_callback *cb)
+-{
+-	enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
+-	struct devlink_trap_policer_item *policer_item;
+-	u32 portid = NETLINK_CB(cb->skb).portid;
+-	struct devlink *devlink;
+-	int start = cb->args[0];
+-	unsigned long index;
+-	int idx = 0;
+-	int err;
+-
+-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
+-		devl_lock(devlink);
+-		list_for_each_entry(policer_item, &devlink->trap_policer_list,
+-				    list) {
+-			if (idx < start) {
+-				idx++;
+-				continue;
+-			}
+-			err = devlink_nl_trap_policer_fill(msg, devlink,
+-							   policer_item, cmd,
+-							   portid,
+-							   cb->nlh->nlmsg_seq,
+-							   NLM_F_MULTI);
+-			if (err) {
+-				devl_unlock(devlink);
+-				devlink_put(devlink);
+-				goto out;
+-			}
+-			idx++;
+-		}
+-		devl_unlock(devlink);
+-		devlink_put(devlink);
+-	}
+-out:
+-	cb->args[0] = idx;
+-	return msg->len;
+-}
+-
+-static int
+-devlink_trap_policer_set(struct devlink *devlink,
+-			 struct devlink_trap_policer_item *policer_item,
+-			 struct genl_info *info)
+-{
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct nlattr **attrs = info->attrs;
+-	u64 rate, burst;
+-	int err;
+-
+-	rate = policer_item->rate;
+-	burst = policer_item->burst;
+-
+-	if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
+-		rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
+-
+-	if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
+-		burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
+-
+-	if (rate < policer_item->policer->min_rate) {
+-		NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
+-		return -EINVAL;
+-	}
+-
+-	if (rate > policer_item->policer->max_rate) {
+-		NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
+-		return -EINVAL;
+-	}
+-
+-	if (burst < policer_item->policer->min_burst) {
+-		NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
+-		return -EINVAL;
+-	}
+-
+-	if (burst > policer_item->policer->max_burst) {
+-		NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
+-		return -EINVAL;
+-	}
+-
+-	err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
+-					     rate, burst, info->extack);
+-	if (err)
+-		return err;
+-
+-	policer_item->rate = rate;
+-	policer_item->burst = burst;
+-
+-	return 0;
+-}
+-
+-static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
+-						struct genl_info *info)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-	struct netlink_ext_ack *extack = info->extack;
+-	struct devlink *devlink = info->user_ptr[0];
+-
+-	if (list_empty(&devlink->trap_policer_list))
+-		return -EOPNOTSUPP;
+-
+-	if (!devlink->ops->trap_policer_set)
+-		return -EOPNOTSUPP;
+-
+-	policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
+-	if (!policer_item) {
+-		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
+-		return -ENOENT;
+-	}
+-
+-	return devlink_trap_policer_set(devlink, policer_item, info);
+-}
+-
+-static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
+-	[DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
+-		DEVLINK_ATTR_TRAP_POLICER_ID },
+-	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
+-						    DEVLINK_PORT_TYPE_IB),
+-	[DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
+-	[DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
+-	[DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
+-						       DEVLINK_ESWITCH_MODE_SWITCHDEV),
+-	[DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
+-	[DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
+-	[DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
+-		NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
+-	[DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
+-	[DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
+-	[DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
+-							DEVLINK_RELOAD_ACTION_MAX),
+-	[DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
+-	[DEVLINK_ATTR_PORT_FLAVOUR] = { .type = NLA_U16 },
+-	[DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 },
+-	[DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 },
+-	[DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 },
+-	[DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 },
+-	[DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING },
+-	[DEVLINK_ATTR_SELFTESTS] = { .type = NLA_NESTED },
+-};
+-
+-static const struct genl_small_ops devlink_nl_ops[] = {
+-	{
+-		.cmd = DEVLINK_CMD_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_get_doit,
+-		.dumpit = devlink_nl_cmd_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_port_get_doit,
+-		.dumpit = devlink_nl_cmd_port_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_port_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RATE_GET,
+-		.doit = devlink_nl_cmd_rate_get_doit,
+-		.dumpit = devlink_nl_cmd_rate_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RATE_SET,
+-		.doit = devlink_nl_cmd_rate_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RATE_NEW,
+-		.doit = devlink_nl_cmd_rate_new_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RATE_DEL,
+-		.doit = devlink_nl_cmd_rate_del_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_SPLIT,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_port_split_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_UNSPLIT,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_port_unsplit_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_NEW,
+-		.doit = devlink_nl_cmd_port_new_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_DEL,
+-		.doit = devlink_nl_cmd_port_del_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_LINECARD_GET,
+-		.doit = devlink_nl_cmd_linecard_get_doit,
+-		.dumpit = devlink_nl_cmd_linecard_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_LINECARD_SET,
+-		.doit = devlink_nl_cmd_linecard_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_get_doit,
+-		.dumpit = devlink_nl_cmd_sb_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_POOL_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_pool_get_doit,
+-		.dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_POOL_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_pool_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_port_pool_get_doit,
+-		.dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_port_pool_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
+-		.dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_occ_snapshot_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_sb_occ_max_clear_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_ESWITCH_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_eswitch_get_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_ESWITCH_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_eswitch_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_dpipe_table_get,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_dpipe_entries_get,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_dpipe_headers_get,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_dpipe_table_counters_set,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RESOURCE_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_resource_set,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RESOURCE_DUMP,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_resource_dump,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_RELOAD,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_reload,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PARAM_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_param_get_doit,
+-		.dumpit = devlink_nl_cmd_param_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PARAM_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_param_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_PARAM_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_port_param_get_doit,
+-		.dumpit = devlink_nl_cmd_port_param_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_PORT_PARAM_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_port_param_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_REGION_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_region_get_doit,
+-		.dumpit = devlink_nl_cmd_region_get_dumpit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_REGION_NEW,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_region_new,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_REGION_DEL,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_region_del,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_REGION_READ,
+-		.validate = GENL_DONT_VALIDATE_STRICT |
+-			    GENL_DONT_VALIDATE_DUMP_STRICT,
+-		.dumpit = devlink_nl_cmd_region_read_dumpit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_INFO_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_info_get_doit,
+-		.dumpit = devlink_nl_cmd_info_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_health_reporter_get_doit,
+-		.dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_health_reporter_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_health_reporter_recover_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_health_reporter_diagnose_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
+-		.validate = GENL_DONT_VALIDATE_STRICT |
+-			    GENL_DONT_VALIDATE_DUMP_STRICT,
+-		.dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_health_reporter_test_doit,
+-		.flags = GENL_ADMIN_PERM,
+-		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_FLASH_UPDATE,
+-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+-		.doit = devlink_nl_cmd_flash_update,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_TRAP_GET,
+-		.doit = devlink_nl_cmd_trap_get_doit,
+-		.dumpit = devlink_nl_cmd_trap_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_TRAP_SET,
+-		.doit = devlink_nl_cmd_trap_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_TRAP_GROUP_GET,
+-		.doit = devlink_nl_cmd_trap_group_get_doit,
+-		.dumpit = devlink_nl_cmd_trap_group_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_TRAP_GROUP_SET,
+-		.doit = devlink_nl_cmd_trap_group_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_TRAP_POLICER_GET,
+-		.doit = devlink_nl_cmd_trap_policer_get_doit,
+-		.dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_TRAP_POLICER_SET,
+-		.doit = devlink_nl_cmd_trap_policer_set_doit,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SELFTESTS_GET,
+-		.doit = devlink_nl_cmd_selftests_get_doit,
+-		.dumpit = devlink_nl_cmd_selftests_get_dumpit
+-		/* can be retrieved by unprivileged users */
+-	},
+-	{
+-		.cmd = DEVLINK_CMD_SELFTESTS_RUN,
+-		.doit = devlink_nl_cmd_selftests_run,
+-		.flags = GENL_ADMIN_PERM,
+-	},
+-};
+-
+-static struct genl_family devlink_nl_family __ro_after_init = {
+-	.name		= DEVLINK_GENL_NAME,
+-	.version	= DEVLINK_GENL_VERSION,
+-	.maxattr	= DEVLINK_ATTR_MAX,
+-	.policy = devlink_nl_policy,
+-	.netnsok	= true,
+-	.parallel_ops	= true,
+-	.pre_doit	= devlink_nl_pre_doit,
+-	.post_doit	= devlink_nl_post_doit,
+-	.module		= THIS_MODULE,
+-	.small_ops	= devlink_nl_ops,
+-	.n_small_ops	= ARRAY_SIZE(devlink_nl_ops),
+-	.resv_start_op	= DEVLINK_CMD_SELFTESTS_RUN + 1,
+-	.mcgrps		= devlink_nl_mcgrps,
+-	.n_mcgrps	= ARRAY_SIZE(devlink_nl_mcgrps),
+-};
+-
+-static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
+-{
+-	const struct devlink_reload_combination *comb;
+-	int i;
+-
+-	if (!devlink_reload_supported(ops)) {
+-		if (WARN_ON(ops->reload_actions))
+-			return false;
+-		return true;
+-	}
+-
+-	if (WARN_ON(!ops->reload_actions ||
+-		    ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
+-		    ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
+-		return false;
+-
+-	if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
+-		    ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
+-		return false;
+-
+-	for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
+-		comb = &devlink_reload_invalid_combinations[i];
+-		if (ops->reload_actions == BIT(comb->action) &&
+-		    ops->reload_limits == BIT(comb->limit))
+-			return false;
+-	}
+-	return true;
+-}
+-
+-/**
+- *	devlink_set_features - Set devlink supported features
+- *
+- *	@devlink: devlink
+- *	@features: devlink support features
+- *
+- *	This interface allows us to set reload ops separatelly from
+- *	the devlink_alloc.
+- */
+-void devlink_set_features(struct devlink *devlink, u64 features)
+-{
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	WARN_ON(features & DEVLINK_F_RELOAD &&
+-		!devlink_reload_supported(devlink->ops));
+-	devlink->features = features;
+-}
+-EXPORT_SYMBOL_GPL(devlink_set_features);
+-
+-/**
+- *	devlink_alloc_ns - Allocate new devlink instance resources
+- *	in specific namespace
+- *
+- *	@ops: ops
+- *	@priv_size: size of user private data
+- *	@net: net namespace
+- *	@dev: parent device
+- *
+- *	Allocate new devlink instance resources, including devlink index
+- *	and name.
+- */
+-struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
+-				 size_t priv_size, struct net *net,
+-				 struct device *dev)
+-{
+-	struct devlink *devlink;
+-	static u32 last_id;
+-	int ret;
+-
+-	WARN_ON(!ops || !dev);
+-	if (!devlink_reload_actions_valid(ops))
+-		return NULL;
+-
+-	devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
+-	if (!devlink)
+-		return NULL;
+-
+-	ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
+-			      &last_id, GFP_KERNEL);
+-	if (ret < 0) {
+-		kfree(devlink);
+-		return NULL;
+-	}
+-
+-	devlink->dev = dev;
+-	devlink->ops = ops;
+-	xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
+-	write_pnet(&devlink->_net, net);
+-	INIT_LIST_HEAD(&devlink->port_list);
+-	INIT_LIST_HEAD(&devlink->rate_list);
+-	INIT_LIST_HEAD(&devlink->linecard_list);
+-	INIT_LIST_HEAD(&devlink->sb_list);
+-	INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
+-	INIT_LIST_HEAD(&devlink->resource_list);
+-	INIT_LIST_HEAD(&devlink->param_list);
+-	INIT_LIST_HEAD(&devlink->region_list);
+-	INIT_LIST_HEAD(&devlink->reporter_list);
+-	INIT_LIST_HEAD(&devlink->trap_list);
+-	INIT_LIST_HEAD(&devlink->trap_group_list);
+-	INIT_LIST_HEAD(&devlink->trap_policer_list);
+-	lockdep_register_key(&devlink->lock_key);
+-	mutex_init(&devlink->lock);
+-	lockdep_set_class(&devlink->lock, &devlink->lock_key);
+-	mutex_init(&devlink->reporters_lock);
+-	mutex_init(&devlink->linecards_lock);
+-	refcount_set(&devlink->refcount, 1);
+-	init_completion(&devlink->comp);
+-
+-	return devlink;
+-}
+-EXPORT_SYMBOL_GPL(devlink_alloc_ns);
+-
+-static void
+-devlink_trap_policer_notify(struct devlink *devlink,
+-			    const struct devlink_trap_policer_item *policer_item,
+-			    enum devlink_command cmd);
+-static void
+-devlink_trap_group_notify(struct devlink *devlink,
+-			  const struct devlink_trap_group_item *group_item,
+-			  enum devlink_command cmd);
+-static void devlink_trap_notify(struct devlink *devlink,
+-				const struct devlink_trap_item *trap_item,
+-				enum devlink_command cmd);
+-
+-static void devlink_notify_register(struct devlink *devlink)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-	struct devlink_trap_group_item *group_item;
+-	struct devlink_param_item *param_item;
+-	struct devlink_trap_item *trap_item;
+-	struct devlink_port *devlink_port;
+-	struct devlink_linecard *linecard;
+-	struct devlink_rate *rate_node;
+-	struct devlink_region *region;
+-
+-	devlink_notify(devlink, DEVLINK_CMD_NEW);
+-	list_for_each_entry(linecard, &devlink->linecard_list, list)
+-		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-
+-	list_for_each_entry(devlink_port, &devlink->port_list, list)
+-		devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
+-
+-	list_for_each_entry(policer_item, &devlink->trap_policer_list, list)
+-		devlink_trap_policer_notify(devlink, policer_item,
+-					    DEVLINK_CMD_TRAP_POLICER_NEW);
+-
+-	list_for_each_entry(group_item, &devlink->trap_group_list, list)
+-		devlink_trap_group_notify(devlink, group_item,
+-					  DEVLINK_CMD_TRAP_GROUP_NEW);
+-
+-	list_for_each_entry(trap_item, &devlink->trap_list, list)
+-		devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
+-
+-	list_for_each_entry(rate_node, &devlink->rate_list, list)
+-		devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
+-
+-	list_for_each_entry(region, &devlink->region_list, list)
+-		devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
+-
+-	list_for_each_entry(param_item, &devlink->param_list, list)
+-		devlink_param_notify(devlink, 0, param_item,
+-				     DEVLINK_CMD_PARAM_NEW);
+-}
+-
+-static void devlink_notify_unregister(struct devlink *devlink)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-	struct devlink_trap_group_item *group_item;
+-	struct devlink_param_item *param_item;
+-	struct devlink_trap_item *trap_item;
+-	struct devlink_port *devlink_port;
+-	struct devlink_rate *rate_node;
+-	struct devlink_region *region;
+-
+-	list_for_each_entry_reverse(param_item, &devlink->param_list, list)
+-		devlink_param_notify(devlink, 0, param_item,
+-				     DEVLINK_CMD_PARAM_DEL);
+-
+-	list_for_each_entry_reverse(region, &devlink->region_list, list)
+-		devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
+-
+-	list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
+-		devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
+-
+-	list_for_each_entry_reverse(trap_item, &devlink->trap_list, list)
+-		devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
+-
+-	list_for_each_entry_reverse(group_item, &devlink->trap_group_list, list)
+-		devlink_trap_group_notify(devlink, group_item,
+-					  DEVLINK_CMD_TRAP_GROUP_DEL);
+-	list_for_each_entry_reverse(policer_item, &devlink->trap_policer_list,
+-				    list)
+-		devlink_trap_policer_notify(devlink, policer_item,
+-					    DEVLINK_CMD_TRAP_POLICER_DEL);
+-
+-	list_for_each_entry_reverse(devlink_port, &devlink->port_list, list)
+-		devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
+-	devlink_notify(devlink, DEVLINK_CMD_DEL);
+-}
+-
+-/**
+- *	devlink_register - Register devlink instance
+- *
+- *	@devlink: devlink
+- */
+-void devlink_register(struct devlink *devlink)
+-{
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-	/* Make sure that we are in .probe() routine */
+-
+-	xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
+-	devlink_notify_register(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_register);
+-
+-/**
+- *	devlink_unregister - Unregister devlink instance
+- *
+- *	@devlink: devlink
+- */
+-void devlink_unregister(struct devlink *devlink)
+-{
+-	ASSERT_DEVLINK_REGISTERED(devlink);
+-	/* Make sure that we are in .remove() routine */
+-
+-	xa_set_mark(&devlinks, devlink->index, DEVLINK_UNREGISTERING);
+-	devlink_put(devlink);
+-	wait_for_completion(&devlink->comp);
+-
+-	devlink_notify_unregister(devlink);
+-	xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
+-	xa_clear_mark(&devlinks, devlink->index, DEVLINK_UNREGISTERING);
+-}
+-EXPORT_SYMBOL_GPL(devlink_unregister);
+-
+-/**
+- *	devlink_free - Free devlink instance resources
+- *
+- *	@devlink: devlink
+- */
+-void devlink_free(struct devlink *devlink)
+-{
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	mutex_destroy(&devlink->linecards_lock);
+-	mutex_destroy(&devlink->reporters_lock);
+-	mutex_destroy(&devlink->lock);
+-	lockdep_unregister_key(&devlink->lock_key);
+-	WARN_ON(!list_empty(&devlink->trap_policer_list));
+-	WARN_ON(!list_empty(&devlink->trap_group_list));
+-	WARN_ON(!list_empty(&devlink->trap_list));
+-	WARN_ON(!list_empty(&devlink->reporter_list));
+-	WARN_ON(!list_empty(&devlink->region_list));
+-	WARN_ON(!list_empty(&devlink->param_list));
+-	WARN_ON(!list_empty(&devlink->resource_list));
+-	WARN_ON(!list_empty(&devlink->dpipe_table_list));
+-	WARN_ON(!list_empty(&devlink->sb_list));
+-	WARN_ON(!list_empty(&devlink->rate_list));
+-	WARN_ON(!list_empty(&devlink->linecard_list));
+-	WARN_ON(!list_empty(&devlink->port_list));
+-
+-	xa_destroy(&devlink->snapshot_ids);
+-	xa_erase(&devlinks, devlink->index);
+-
+-	kfree(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_free);
+-
+-static void devlink_port_type_warn(struct work_struct *work)
+-{
+-	struct devlink_port *port = container_of(to_delayed_work(work),
+-						 struct devlink_port,
+-						 type_warn_dw);
+-	dev_warn(port->devlink->dev, "Type was not set for devlink port.");
+-}
+-
+-static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
+-{
+-	/* Ignore CPU and DSA flavours. */
+-	return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
+-	       devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
+-	       devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
+-}
+-
+-#define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
+-
+-static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
+-{
+-	if (!devlink_port_type_should_warn(devlink_port))
+-		return;
+-	/* Schedule a work to WARN in case driver does not set port
+-	 * type within timeout.
+-	 */
+-	schedule_delayed_work(&devlink_port->type_warn_dw,
+-			      DEVLINK_PORT_TYPE_WARN_TIMEOUT);
+-}
+-
+-static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
+-{
+-	if (!devlink_port_type_should_warn(devlink_port))
+-		return;
+-	cancel_delayed_work_sync(&devlink_port->type_warn_dw);
+-}
+-
+-/**
+- * devlink_port_init() - Init devlink port
+- *
+- * @devlink: devlink
+- * @devlink_port: devlink port
+- *
+- * Initialize essencial stuff that is needed for functions
+- * that may be called before devlink port registration.
+- * Call to this function is optional and not needed
+- * in case the driver does not use such functions.
+- */
+-void devlink_port_init(struct devlink *devlink,
+-		       struct devlink_port *devlink_port)
+-{
+-	if (devlink_port->initialized)
+-		return;
+-	devlink_port->devlink = devlink;
+-	INIT_LIST_HEAD(&devlink_port->region_list);
+-	devlink_port->initialized = true;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_init);
+-
+-/**
+- * devlink_port_fini() - Deinitialize devlink port
+- *
+- * @devlink_port: devlink port
+- *
+- * Deinitialize essencial stuff that is in use for functions
+- * that may be called after devlink port unregistration.
+- * Call to this function is optional and not needed
+- * in case the driver does not use such functions.
+- */
+-void devlink_port_fini(struct devlink_port *devlink_port)
+-{
+-	WARN_ON(!list_empty(&devlink_port->region_list));
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_fini);
+-
+-/**
+- * devl_port_register() - Register devlink port
+- *
+- * @devlink: devlink
+- * @devlink_port: devlink port
+- * @port_index: driver-specific numerical identifier of the port
+- *
+- * Register devlink port with provided port index. User can use
+- * any indexing, even hw-related one. devlink_port structure
+- * is convenient to be embedded inside user driver private structure.
+- * Note that the caller should take care of zeroing the devlink_port
+- * structure.
+- */
+-int devl_port_register(struct devlink *devlink,
+-		       struct devlink_port *devlink_port,
+-		       unsigned int port_index)
+-{
+-	devl_assert_locked(devlink);
+-
+-	if (devlink_port_index_exists(devlink, port_index))
+-		return -EEXIST;
+-
+-	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+-
+-	devlink_port_init(devlink, devlink_port);
+-	devlink_port->registered = true;
+-	devlink_port->index = port_index;
+-	spin_lock_init(&devlink_port->type_lock);
+-	INIT_LIST_HEAD(&devlink_port->reporter_list);
+-	mutex_init(&devlink_port->reporters_lock);
+-	list_add_tail(&devlink_port->list, &devlink->port_list);
+-
+-	INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
+-	devlink_port_type_warn_schedule(devlink_port);
+-	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_port_register);
+-
+-/**
+- *	devlink_port_register - Register devlink port
+- *
+- *	@devlink: devlink
+- *	@devlink_port: devlink port
+- *	@port_index: driver-specific numerical identifier of the port
+- *
+- *	Register devlink port with provided port index. User can use
+- *	any indexing, even hw-related one. devlink_port structure
+- *	is convenient to be embedded inside user driver private structure.
+- *	Note that the caller should take care of zeroing the devlink_port
+- *	structure.
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-int devlink_port_register(struct devlink *devlink,
+-			  struct devlink_port *devlink_port,
+-			  unsigned int port_index)
+-{
+-	int err;
+-
+-	devl_lock(devlink);
+-	err = devl_port_register(devlink, devlink_port, port_index);
+-	devl_unlock(devlink);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_register);
+-
+-/**
+- * devl_port_unregister() - Unregister devlink port
+- *
+- * @devlink_port: devlink port
+- */
+-void devl_port_unregister(struct devlink_port *devlink_port)
+-{
+-	lockdep_assert_held(&devlink_port->devlink->lock);
+-
+-	devlink_port_type_warn_cancel(devlink_port);
+-	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
+-	list_del(&devlink_port->list);
+-	WARN_ON(!list_empty(&devlink_port->reporter_list));
+-	mutex_destroy(&devlink_port->reporters_lock);
+-	devlink_port->registered = false;
+-}
+-EXPORT_SYMBOL_GPL(devl_port_unregister);
+-
+-/**
+- *	devlink_port_unregister - Unregister devlink port
+- *
+- *	@devlink_port: devlink port
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_port_unregister(struct devlink_port *devlink_port)
+-{
+-	struct devlink *devlink = devlink_port->devlink;
+-
+-	devl_lock(devlink);
+-	devl_port_unregister(devlink_port);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_unregister);
+-
+-static void __devlink_port_type_set(struct devlink_port *devlink_port,
+-				    enum devlink_port_type type,
+-				    void *type_dev)
+-{
+-	ASSERT_DEVLINK_PORT_REGISTERED(devlink_port);
+-
+-	devlink_port_type_warn_cancel(devlink_port);
+-	spin_lock_bh(&devlink_port->type_lock);
+-	devlink_port->type = type;
+-	devlink_port->type_dev = type_dev;
+-	spin_unlock_bh(&devlink_port->type_lock);
+-	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
+-}
+-
+-static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
+-					    struct net_device *netdev)
+-{
+-	const struct net_device_ops *ops = netdev->netdev_ops;
+-
+-	/* If driver registers devlink port, it should set devlink port
+-	 * attributes accordingly so the compat functions are called
+-	 * and the original ops are not used.
+-	 */
+-	if (ops->ndo_get_phys_port_name) {
+-		/* Some drivers use the same set of ndos for netdevs
+-		 * that have devlink_port registered and also for
+-		 * those who don't. Make sure that ndo_get_phys_port_name
+-		 * returns -EOPNOTSUPP here in case it is defined.
+-		 * Warn if not.
+-		 */
+-		char name[IFNAMSIZ];
+-		int err;
+-
+-		err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
+-		WARN_ON(err != -EOPNOTSUPP);
+-	}
+-	if (ops->ndo_get_port_parent_id) {
+-		/* Some drivers use the same set of ndos for netdevs
+-		 * that have devlink_port registered and also for
+-		 * those who don't. Make sure that ndo_get_port_parent_id
+-		 * returns -EOPNOTSUPP here in case it is defined.
+-		 * Warn if not.
+-		 */
+-		struct netdev_phys_item_id ppid;
+-		int err;
+-
+-		err = ops->ndo_get_port_parent_id(netdev, &ppid);
+-		WARN_ON(err != -EOPNOTSUPP);
+-	}
+-}
+-
+-/**
+- *	devlink_port_type_eth_set - Set port type to Ethernet
+- *
+- *	@devlink_port: devlink port
+- *	@netdev: related netdevice
+- */
+-void devlink_port_type_eth_set(struct devlink_port *devlink_port,
+-			       struct net_device *netdev)
+-{
+-	if (netdev)
+-		devlink_port_type_netdev_checks(devlink_port, netdev);
+-	else
+-		dev_warn(devlink_port->devlink->dev,
+-			 "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
+-			 devlink_port->index);
+-
+-	__devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
+-
+-/**
+- *	devlink_port_type_ib_set - Set port type to InfiniBand
+- *
+- *	@devlink_port: devlink port
+- *	@ibdev: related IB device
+- */
+-void devlink_port_type_ib_set(struct devlink_port *devlink_port,
+-			      struct ib_device *ibdev)
+-{
+-	__devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
+-
+-/**
+- *	devlink_port_type_clear - Clear port type
+- *
+- *	@devlink_port: devlink port
+- */
+-void devlink_port_type_clear(struct devlink_port *devlink_port)
+-{
+-	__devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
+-	devlink_port_type_warn_schedule(devlink_port);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_type_clear);
+-
+-static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
+-				    enum devlink_port_flavour flavour)
+-{
+-	struct devlink_port_attrs *attrs = &devlink_port->attrs;
+-
+-	devlink_port->attrs_set = true;
+-	attrs->flavour = flavour;
+-	if (attrs->switch_id.id_len) {
+-		devlink_port->switch_port = true;
+-		if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
+-			attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
+-	} else {
+-		devlink_port->switch_port = false;
+-	}
+-	return 0;
+-}
+-
+-/**
+- *	devlink_port_attrs_set - Set port attributes
+- *
+- *	@devlink_port: devlink port
+- *	@attrs: devlink port attrs
+- */
+-void devlink_port_attrs_set(struct devlink_port *devlink_port,
+-			    struct devlink_port_attrs *attrs)
+-{
+-	int ret;
+-
+-	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+-
+-	devlink_port->attrs = *attrs;
+-	ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
+-	if (ret)
+-		return;
+-	WARN_ON(attrs->splittable && attrs->split);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
+-
+-/**
+- *	devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
+- *
+- *	@devlink_port: devlink port
+- *	@controller: associated controller number for the devlink port instance
+- *	@pf: associated PF for the devlink port instance
+- *	@external: indicates if the port is for an external controller
+- */
+-void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
+-				   u16 pf, bool external)
+-{
+-	struct devlink_port_attrs *attrs = &devlink_port->attrs;
+-	int ret;
+-
+-	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+-
+-	ret = __devlink_port_attrs_set(devlink_port,
+-				       DEVLINK_PORT_FLAVOUR_PCI_PF);
+-	if (ret)
+-		return;
+-	attrs->pci_pf.controller = controller;
+-	attrs->pci_pf.pf = pf;
+-	attrs->pci_pf.external = external;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
+-
+-/**
+- *	devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
+- *
+- *	@devlink_port: devlink port
+- *	@controller: associated controller number for the devlink port instance
+- *	@pf: associated PF for the devlink port instance
+- *	@vf: associated VF of a PF for the devlink port instance
+- *	@external: indicates if the port is for an external controller
+- */
+-void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
+-				   u16 pf, u16 vf, bool external)
+-{
+-	struct devlink_port_attrs *attrs = &devlink_port->attrs;
+-	int ret;
+-
+-	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+-
+-	ret = __devlink_port_attrs_set(devlink_port,
+-				       DEVLINK_PORT_FLAVOUR_PCI_VF);
+-	if (ret)
+-		return;
+-	attrs->pci_vf.controller = controller;
+-	attrs->pci_vf.pf = pf;
+-	attrs->pci_vf.vf = vf;
+-	attrs->pci_vf.external = external;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
+-
+-/**
+- *	devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
+- *
+- *	@devlink_port: devlink port
+- *	@controller: associated controller number for the devlink port instance
+- *	@pf: associated PF for the devlink port instance
+- *	@sf: associated SF of a PF for the devlink port instance
+- *	@external: indicates if the port is for an external controller
+- */
+-void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
+-				   u16 pf, u32 sf, bool external)
+-{
+-	struct devlink_port_attrs *attrs = &devlink_port->attrs;
+-	int ret;
+-
+-	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+-
+-	ret = __devlink_port_attrs_set(devlink_port,
+-				       DEVLINK_PORT_FLAVOUR_PCI_SF);
+-	if (ret)
+-		return;
+-	attrs->pci_sf.controller = controller;
+-	attrs->pci_sf.pf = pf;
+-	attrs->pci_sf.sf = sf;
+-	attrs->pci_sf.external = external;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);
+-
+-/**
+- * devl_rate_leaf_create - create devlink rate leaf
+- * @devlink_port: devlink port object to create rate object on
+- * @priv: driver private data
+- *
+- * Create devlink rate object of type leaf on provided @devlink_port.
+- */
+-int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
+-{
+-	struct devlink *devlink = devlink_port->devlink;
+-	struct devlink_rate *devlink_rate;
+-
+-	devl_assert_locked(devlink_port->devlink);
+-
+-	if (WARN_ON(devlink_port->devlink_rate))
+-		return -EBUSY;
+-
+-	devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL);
+-	if (!devlink_rate)
+-		return -ENOMEM;
+-
+-	devlink_rate->type = DEVLINK_RATE_TYPE_LEAF;
+-	devlink_rate->devlink = devlink;
+-	devlink_rate->devlink_port = devlink_port;
+-	devlink_rate->priv = priv;
+-	list_add_tail(&devlink_rate->list, &devlink->rate_list);
+-	devlink_port->devlink_rate = devlink_rate;
+-	devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_rate_leaf_create);
+-
+-/**
+- * devl_rate_leaf_destroy - destroy devlink rate leaf
+- *
+- * @devlink_port: devlink port linked to the rate object
+- *
+- * Destroy the devlink rate object of type leaf on provided @devlink_port.
+- */
+-void devl_rate_leaf_destroy(struct devlink_port *devlink_port)
+-{
+-	struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
+-
+-	devl_assert_locked(devlink_port->devlink);
+-	if (!devlink_rate)
+-		return;
+-
+-	devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
+-	if (devlink_rate->parent)
+-		refcount_dec(&devlink_rate->parent->refcnt);
+-	list_del(&devlink_rate->list);
+-	devlink_port->devlink_rate = NULL;
+-	kfree(devlink_rate);
+-}
+-EXPORT_SYMBOL_GPL(devl_rate_leaf_destroy);
+-
+-/**
+- * devl_rate_nodes_destroy - destroy all devlink rate nodes on device
+- * @devlink: devlink instance
+- *
+- * Unset parent for all rate objects and destroy all rate nodes
+- * on specified device.
+- */
+-void devl_rate_nodes_destroy(struct devlink *devlink)
+-{
+-	static struct devlink_rate *devlink_rate, *tmp;
+-	const struct devlink_ops *ops = devlink->ops;
+-
+-	devl_assert_locked(devlink);
+-
+-	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
+-		if (!devlink_rate->parent)
+-			continue;
+-
+-		refcount_dec(&devlink_rate->parent->refcnt);
+-		if (devlink_rate_is_leaf(devlink_rate))
+-			ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
+-						  NULL, NULL);
+-		else if (devlink_rate_is_node(devlink_rate))
+-			ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv,
+-						  NULL, NULL);
+-	}
+-	list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
+-		if (devlink_rate_is_node(devlink_rate)) {
+-			ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
+-			list_del(&devlink_rate->list);
+-			kfree(devlink_rate->name);
+-			kfree(devlink_rate);
+-		}
+-	}
+-}
+-EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy);
+-
+-/**
+- *	devlink_port_linecard_set - Link port with a linecard
+- *
+- *	@devlink_port: devlink port
+- *	@linecard: devlink linecard
+- */
+-void devlink_port_linecard_set(struct devlink_port *devlink_port,
+-			       struct devlink_linecard *linecard)
+-{
+-	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+-
+-	devlink_port->linecard = linecard;
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_linecard_set);
+-
+-static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
+-					     char *name, size_t len)
+-{
+-	struct devlink_port_attrs *attrs = &devlink_port->attrs;
+-	int n = 0;
+-
+-	if (!devlink_port->attrs_set)
+-		return -EOPNOTSUPP;
+-
+-	switch (attrs->flavour) {
+-	case DEVLINK_PORT_FLAVOUR_PHYSICAL:
+-		if (devlink_port->linecard)
+-			n = snprintf(name, len, "l%u",
+-				     devlink_port->linecard->index);
+-		if (n < len)
+-			n += snprintf(name + n, len - n, "p%u",
+-				      attrs->phys.port_number);
+-		if (n < len && attrs->split)
+-			n += snprintf(name + n, len - n, "s%u",
+-				      attrs->phys.split_subport_number);
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_CPU:
+-	case DEVLINK_PORT_FLAVOUR_DSA:
+-	case DEVLINK_PORT_FLAVOUR_UNUSED:
+-		/* As CPU and DSA ports do not have a netdevice associated
+-		 * case should not ever happen.
+-		 */
+-		WARN_ON(1);
+-		return -EINVAL;
+-	case DEVLINK_PORT_FLAVOUR_PCI_PF:
+-		if (attrs->pci_pf.external) {
+-			n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
+-			if (n >= len)
+-				return -EINVAL;
+-			len -= n;
+-			name += n;
+-		}
+-		n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_PCI_VF:
+-		if (attrs->pci_vf.external) {
+-			n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
+-			if (n >= len)
+-				return -EINVAL;
+-			len -= n;
+-			name += n;
+-		}
+-		n = snprintf(name, len, "pf%uvf%u",
+-			     attrs->pci_vf.pf, attrs->pci_vf.vf);
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_PCI_SF:
+-		if (attrs->pci_sf.external) {
+-			n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
+-			if (n >= len)
+-				return -EINVAL;
+-			len -= n;
+-			name += n;
+-		}
+-		n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
+-			     attrs->pci_sf.sf);
+-		break;
+-	case DEVLINK_PORT_FLAVOUR_VIRTUAL:
+-		return -EOPNOTSUPP;
+-	}
+-
+-	if (n >= len)
+-		return -EINVAL;
+-
+-	return 0;
+-}
+-
+-static int devlink_linecard_types_init(struct devlink_linecard *linecard)
+-{
+-	struct devlink_linecard_type *linecard_type;
+-	unsigned int count;
+-	int i;
+-
+-	count = linecard->ops->types_count(linecard, linecard->priv);
+-	linecard->types = kmalloc_array(count, sizeof(*linecard_type),
+-					GFP_KERNEL);
+-	if (!linecard->types)
+-		return -ENOMEM;
+-	linecard->types_count = count;
+-
+-	for (i = 0; i < count; i++) {
+-		linecard_type = &linecard->types[i];
+-		linecard->ops->types_get(linecard, linecard->priv, i,
+-					 &linecard_type->type,
+-					 &linecard_type->priv);
+-	}
+-	return 0;
+-}
+-
+-static void devlink_linecard_types_fini(struct devlink_linecard *linecard)
+-{
+-	kfree(linecard->types);
+-}
+-
+-/**
+- *	devlink_linecard_create - Create devlink linecard
+- *
+- *	@devlink: devlink
+- *	@linecard_index: driver-specific numerical identifier of the linecard
+- *	@ops: linecards ops
+- *	@priv: user priv pointer
+- *
+- *	Create devlink linecard instance with provided linecard index.
+- *	Caller can use any indexing, even hw-related one.
+- *
+- *	Return: Line card structure or an ERR_PTR() encoded error code.
+- */
+-struct devlink_linecard *
+-devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index,
+-			const struct devlink_linecard_ops *ops, void *priv)
+-{
+-	struct devlink_linecard *linecard;
+-	int err;
+-
+-	if (WARN_ON(!ops || !ops->provision || !ops->unprovision ||
+-		    !ops->types_count || !ops->types_get))
+-		return ERR_PTR(-EINVAL);
+-
+-	mutex_lock(&devlink->linecards_lock);
+-	if (devlink_linecard_index_exists(devlink, linecard_index)) {
+-		mutex_unlock(&devlink->linecards_lock);
+-		return ERR_PTR(-EEXIST);
+-	}
+-
+-	linecard = kzalloc(sizeof(*linecard), GFP_KERNEL);
+-	if (!linecard) {
+-		mutex_unlock(&devlink->linecards_lock);
+-		return ERR_PTR(-ENOMEM);
+-	}
+-
+-	linecard->devlink = devlink;
+-	linecard->index = linecard_index;
+-	linecard->ops = ops;
+-	linecard->priv = priv;
+-	linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
+-	mutex_init(&linecard->state_lock);
+-
+-	err = devlink_linecard_types_init(linecard);
+-	if (err) {
+-		mutex_destroy(&linecard->state_lock);
+-		kfree(linecard);
+-		mutex_unlock(&devlink->linecards_lock);
+-		return ERR_PTR(err);
+-	}
+-
+-	list_add_tail(&linecard->list, &devlink->linecard_list);
+-	refcount_set(&linecard->refcount, 1);
+-	mutex_unlock(&devlink->linecards_lock);
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	return linecard;
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_create);
+-
+-/**
+- *	devlink_linecard_destroy - Destroy devlink linecard
+- *
+- *	@linecard: devlink linecard
+- */
+-void devlink_linecard_destroy(struct devlink_linecard *linecard)
+-{
+-	struct devlink *devlink = linecard->devlink;
+-
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_DEL);
+-	mutex_lock(&devlink->linecards_lock);
+-	list_del(&linecard->list);
+-	devlink_linecard_types_fini(linecard);
+-	mutex_unlock(&devlink->linecards_lock);
+-	devlink_linecard_put(linecard);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_destroy);
+-
+-/**
+- *	devlink_linecard_provision_set - Set provisioning on linecard
+- *
+- *	@linecard: devlink linecard
+- *	@type: linecard type
+- *
+- *	This is either called directly from the provision() op call or
+- *	as a result of the provision() op call asynchronously.
+- */
+-void devlink_linecard_provision_set(struct devlink_linecard *linecard,
+-				    const char *type)
+-{
+-	mutex_lock(&linecard->state_lock);
+-	WARN_ON(linecard->type && strcmp(linecard->type, type));
+-	linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
+-	linecard->type = type;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_provision_set);
+-
+-/**
+- *	devlink_linecard_provision_clear - Clear provisioning on linecard
+- *
+- *	@linecard: devlink linecard
+- *
+- *	This is either called directly from the unprovision() op call or
+- *	as a result of the unprovision() op call asynchronously.
+- */
+-void devlink_linecard_provision_clear(struct devlink_linecard *linecard)
+-{
+-	mutex_lock(&linecard->state_lock);
+-	WARN_ON(linecard->nested_devlink);
+-	linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
+-	linecard->type = NULL;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear);
+-
+-/**
+- *	devlink_linecard_provision_fail - Fail provisioning on linecard
+- *
+- *	@linecard: devlink linecard
+- *
+- *	This is either called directly from the provision() op call or
+- *	as a result of the provision() op call asynchronously.
+- */
+-void devlink_linecard_provision_fail(struct devlink_linecard *linecard)
+-{
+-	mutex_lock(&linecard->state_lock);
+-	WARN_ON(linecard->nested_devlink);
+-	linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING_FAILED;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_provision_fail);
+-
+-/**
+- *	devlink_linecard_activate - Set linecard active
+- *
+- *	@linecard: devlink linecard
+- */
+-void devlink_linecard_activate(struct devlink_linecard *linecard)
+-{
+-	mutex_lock(&linecard->state_lock);
+-	WARN_ON(linecard->state != DEVLINK_LINECARD_STATE_PROVISIONED);
+-	linecard->state = DEVLINK_LINECARD_STATE_ACTIVE;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_activate);
+-
+-/**
+- *	devlink_linecard_deactivate - Set linecard inactive
+- *
+- *	@linecard: devlink linecard
+- */
+-void devlink_linecard_deactivate(struct devlink_linecard *linecard)
+-{
+-	mutex_lock(&linecard->state_lock);
+-	switch (linecard->state) {
+-	case DEVLINK_LINECARD_STATE_ACTIVE:
+-		linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
+-		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-		break;
+-	case DEVLINK_LINECARD_STATE_UNPROVISIONING:
+-		/* Line card is being deactivated as part
+-		 * of unprovisioning flow.
+-		 */
+-		break;
+-	default:
+-		WARN_ON(1);
+-		break;
+-	}
+-	mutex_unlock(&linecard->state_lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_deactivate);
+-
+-/**
+- *	devlink_linecard_nested_dl_set - Attach/detach nested devlink
+- *					 instance to linecard.
+- *
+- *	@linecard: devlink linecard
+- *	@nested_devlink: devlink instance to attach or NULL to detach
+- */
+-void devlink_linecard_nested_dl_set(struct devlink_linecard *linecard,
+-				    struct devlink *nested_devlink)
+-{
+-	mutex_lock(&linecard->state_lock);
+-	linecard->nested_devlink = nested_devlink;
+-	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
+-	mutex_unlock(&linecard->state_lock);
+-}
+-EXPORT_SYMBOL_GPL(devlink_linecard_nested_dl_set);
+-
+-int devl_sb_register(struct devlink *devlink, unsigned int sb_index,
+-		     u32 size, u16 ingress_pools_count,
+-		     u16 egress_pools_count, u16 ingress_tc_count,
+-		     u16 egress_tc_count)
+-{
+-	struct devlink_sb *devlink_sb;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	if (devlink_sb_index_exists(devlink, sb_index))
+-		return -EEXIST;
+-
+-	devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
+-	if (!devlink_sb)
+-		return -ENOMEM;
+-	devlink_sb->index = sb_index;
+-	devlink_sb->size = size;
+-	devlink_sb->ingress_pools_count = ingress_pools_count;
+-	devlink_sb->egress_pools_count = egress_pools_count;
+-	devlink_sb->ingress_tc_count = ingress_tc_count;
+-	devlink_sb->egress_tc_count = egress_tc_count;
+-	list_add_tail(&devlink_sb->list, &devlink->sb_list);
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_sb_register);
+-
+-int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
+-			u32 size, u16 ingress_pools_count,
+-			u16 egress_pools_count, u16 ingress_tc_count,
+-			u16 egress_tc_count)
+-{
+-	int err;
+-
+-	devl_lock(devlink);
+-	err = devl_sb_register(devlink, sb_index, size, ingress_pools_count,
+-			       egress_pools_count, ingress_tc_count,
+-			       egress_tc_count);
+-	devl_unlock(devlink);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_sb_register);
+-
+-void devl_sb_unregister(struct devlink *devlink, unsigned int sb_index)
+-{
+-	struct devlink_sb *devlink_sb;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
+-	WARN_ON(!devlink_sb);
+-	list_del(&devlink_sb->list);
+-	kfree(devlink_sb);
+-}
+-EXPORT_SYMBOL_GPL(devl_sb_unregister);
+-
+-void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
+-{
+-	devl_lock(devlink);
+-	devl_sb_unregister(devlink, sb_index);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_sb_unregister);
+-
+-/**
+- * devl_dpipe_headers_register - register dpipe headers
+- *
+- * @devlink: devlink
+- * @dpipe_headers: dpipe header array
+- *
+- * Register the headers supported by hardware.
+- */
+-void devl_dpipe_headers_register(struct devlink *devlink,
+-				 struct devlink_dpipe_headers *dpipe_headers)
+-{
+-	lockdep_assert_held(&devlink->lock);
+-
+-	devlink->dpipe_headers = dpipe_headers;
+-}
+-EXPORT_SYMBOL_GPL(devl_dpipe_headers_register);
+-
+-/**
+- * devl_dpipe_headers_unregister - unregister dpipe headers
+- *
+- * @devlink: devlink
+- *
+- * Unregister the headers supported by hardware.
+- */
+-void devl_dpipe_headers_unregister(struct devlink *devlink)
+-{
+-	lockdep_assert_held(&devlink->lock);
+-
+-	devlink->dpipe_headers = NULL;
+-}
+-EXPORT_SYMBOL_GPL(devl_dpipe_headers_unregister);
+-
+-/**
+- *	devlink_dpipe_table_counter_enabled - check if counter allocation
+- *					      required
+- *	@devlink: devlink
+- *	@table_name: tables name
+- *
+- *	Used by driver to check if counter allocation is required.
+- *	After counter allocation is turned on the table entries
+- *	are updated to include counter statistics.
+- *
+- *	After that point on the driver must respect the counter
+- *	state so that each entry added to the table is added
+- *	with a counter.
+- */
+-bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
+-					 const char *table_name)
+-{
+-	struct devlink_dpipe_table *table;
+-	bool enabled;
+-
+-	rcu_read_lock();
+-	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
+-					 table_name, devlink);
+-	enabled = false;
+-	if (table)
+-		enabled = table->counters_enabled;
+-	rcu_read_unlock();
+-	return enabled;
+-}
+-EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
+-
+-/**
+- * devl_dpipe_table_register - register dpipe table
+- *
+- * @devlink: devlink
+- * @table_name: table name
+- * @table_ops: table ops
+- * @priv: priv
+- * @counter_control_extern: external control for counters
+- */
+-int devl_dpipe_table_register(struct devlink *devlink,
+-			      const char *table_name,
+-			      struct devlink_dpipe_table_ops *table_ops,
+-			      void *priv, bool counter_control_extern)
+-{
+-	struct devlink_dpipe_table *table;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	if (WARN_ON(!table_ops->size_get))
+-		return -EINVAL;
+-
+-	if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
+-				     devlink))
+-		return -EEXIST;
+-
+-	table = kzalloc(sizeof(*table), GFP_KERNEL);
+-	if (!table)
+-		return -ENOMEM;
+-
+-	table->name = table_name;
+-	table->table_ops = table_ops;
+-	table->priv = priv;
+-	table->counter_control_extern = counter_control_extern;
+-
+-	list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_dpipe_table_register);
+-
+-/**
+- * devl_dpipe_table_unregister - unregister dpipe table
+- *
+- * @devlink: devlink
+- * @table_name: table name
+- */
+-void devl_dpipe_table_unregister(struct devlink *devlink,
+-				 const char *table_name)
+-{
+-	struct devlink_dpipe_table *table;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
+-					 table_name, devlink);
+-	if (!table)
+-		return;
+-	list_del_rcu(&table->list);
+-	kfree_rcu(table, rcu);
+-}
+-EXPORT_SYMBOL_GPL(devl_dpipe_table_unregister);
+-
+-/**
+- * devl_resource_register - devlink resource register
+- *
+- * @devlink: devlink
+- * @resource_name: resource's name
+- * @resource_size: resource's size
+- * @resource_id: resource's id
+- * @parent_resource_id: resource's parent id
+- * @size_params: size parameters
+- *
+- * Generic resources should reuse the same names across drivers.
+- * Please see the generic resources list at:
+- * Documentation/networking/devlink/devlink-resource.rst
+- */
+-int devl_resource_register(struct devlink *devlink,
+-			   const char *resource_name,
+-			   u64 resource_size,
+-			   u64 resource_id,
+-			   u64 parent_resource_id,
+-			   const struct devlink_resource_size_params *size_params)
+-{
+-	struct devlink_resource *resource;
+-	struct list_head *resource_list;
+-	bool top_hierarchy;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
+-
+-	resource = devlink_resource_find(devlink, NULL, resource_id);
+-	if (resource)
+-		return -EINVAL;
+-
+-	resource = kzalloc(sizeof(*resource), GFP_KERNEL);
+-	if (!resource)
+-		return -ENOMEM;
+-
+-	if (top_hierarchy) {
+-		resource_list = &devlink->resource_list;
+-	} else {
+-		struct devlink_resource *parent_resource;
+-
+-		parent_resource = devlink_resource_find(devlink, NULL,
+-							parent_resource_id);
+-		if (parent_resource) {
+-			resource_list = &parent_resource->resource_list;
+-			resource->parent = parent_resource;
+-		} else {
+-			kfree(resource);
+-			return -EINVAL;
+-		}
+-	}
+-
+-	resource->name = resource_name;
+-	resource->size = resource_size;
+-	resource->size_new = resource_size;
+-	resource->id = resource_id;
+-	resource->size_valid = true;
+-	memcpy(&resource->size_params, size_params,
+-	       sizeof(resource->size_params));
+-	INIT_LIST_HEAD(&resource->resource_list);
+-	list_add_tail(&resource->list, resource_list);
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_resource_register);
+-
+-/**
+- *	devlink_resource_register - devlink resource register
+- *
+- *	@devlink: devlink
+- *	@resource_name: resource's name
+- *	@resource_size: resource's size
+- *	@resource_id: resource's id
+- *	@parent_resource_id: resource's parent id
+- *	@size_params: size parameters
+- *
+- *	Generic resources should reuse the same names across drivers.
+- *	Please see the generic resources list at:
+- *	Documentation/networking/devlink/devlink-resource.rst
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-int devlink_resource_register(struct devlink *devlink,
+-			      const char *resource_name,
+-			      u64 resource_size,
+-			      u64 resource_id,
+-			      u64 parent_resource_id,
+-			      const struct devlink_resource_size_params *size_params)
+-{
+-	int err;
+-
+-	devl_lock(devlink);
+-	err = devl_resource_register(devlink, resource_name, resource_size,
+-				     resource_id, parent_resource_id, size_params);
+-	devl_unlock(devlink);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_resource_register);
+-
+-static void devlink_resource_unregister(struct devlink *devlink,
+-					struct devlink_resource *resource)
+-{
+-	struct devlink_resource *tmp, *child_resource;
+-
+-	list_for_each_entry_safe(child_resource, tmp, &resource->resource_list,
+-				 list) {
+-		devlink_resource_unregister(devlink, child_resource);
+-		list_del(&child_resource->list);
+-		kfree(child_resource);
+-	}
+-}
+-
+-/**
+- * devl_resources_unregister - free all resources
+- *
+- * @devlink: devlink
+- */
+-void devl_resources_unregister(struct devlink *devlink)
+-{
+-	struct devlink_resource *tmp, *child_resource;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list,
+-				 list) {
+-		devlink_resource_unregister(devlink, child_resource);
+-		list_del(&child_resource->list);
+-		kfree(child_resource);
+-	}
+-}
+-EXPORT_SYMBOL_GPL(devl_resources_unregister);
+-
+-/**
+- *	devlink_resources_unregister - free all resources
+- *
+- *	@devlink: devlink
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_resources_unregister(struct devlink *devlink)
+-{
+-	devl_lock(devlink);
+-	devl_resources_unregister(devlink);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_resources_unregister);
+-
+-/**
+- * devl_resource_size_get - get and update size
+- *
+- * @devlink: devlink
+- * @resource_id: the requested resource id
+- * @p_resource_size: ptr to update
+- */
+-int devl_resource_size_get(struct devlink *devlink,
+-			   u64 resource_id,
+-			   u64 *p_resource_size)
+-{
+-	struct devlink_resource *resource;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	resource = devlink_resource_find(devlink, NULL, resource_id);
+-	if (!resource)
+-		return -EINVAL;
+-	*p_resource_size = resource->size_new;
+-	resource->size = resource->size_new;
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_resource_size_get);
+-
+-/**
+- * devl_dpipe_table_resource_set - set the resource id
+- *
+- * @devlink: devlink
+- * @table_name: table name
+- * @resource_id: resource id
+- * @resource_units: number of resource's units consumed per table's entry
+- */
+-int devl_dpipe_table_resource_set(struct devlink *devlink,
+-				  const char *table_name, u64 resource_id,
+-				  u64 resource_units)
+-{
+-	struct devlink_dpipe_table *table;
+-
+-	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
+-					 table_name, devlink);
+-	if (!table)
+-		return -EINVAL;
+-
+-	table->resource_id = resource_id;
+-	table->resource_units = resource_units;
+-	table->resource_valid = true;
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devl_dpipe_table_resource_set);
+-
+-/**
+- * devl_resource_occ_get_register - register occupancy getter
+- *
+- * @devlink: devlink
+- * @resource_id: resource id
+- * @occ_get: occupancy getter callback
+- * @occ_get_priv: occupancy getter callback priv
+- */
+-void devl_resource_occ_get_register(struct devlink *devlink,
+-				    u64 resource_id,
+-				    devlink_resource_occ_get_t *occ_get,
+-				    void *occ_get_priv)
+-{
+-	struct devlink_resource *resource;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	resource = devlink_resource_find(devlink, NULL, resource_id);
+-	if (WARN_ON(!resource))
+-		return;
+-	WARN_ON(resource->occ_get);
+-
+-	resource->occ_get = occ_get;
+-	resource->occ_get_priv = occ_get_priv;
+-}
+-EXPORT_SYMBOL_GPL(devl_resource_occ_get_register);
+-
+-/**
+- *	devlink_resource_occ_get_register - register occupancy getter
+- *
+- *	@devlink: devlink
+- *	@resource_id: resource id
+- *	@occ_get: occupancy getter callback
+- *	@occ_get_priv: occupancy getter callback priv
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_resource_occ_get_register(struct devlink *devlink,
+-				       u64 resource_id,
+-				       devlink_resource_occ_get_t *occ_get,
+-				       void *occ_get_priv)
+-{
+-	devl_lock(devlink);
+-	devl_resource_occ_get_register(devlink, resource_id,
+-				       occ_get, occ_get_priv);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
+-
+-/**
+- * devl_resource_occ_get_unregister - unregister occupancy getter
+- *
+- * @devlink: devlink
+- * @resource_id: resource id
+- */
+-void devl_resource_occ_get_unregister(struct devlink *devlink,
+-				      u64 resource_id)
+-{
+-	struct devlink_resource *resource;
+-
+-	lockdep_assert_held(&devlink->lock);
+-
+-	resource = devlink_resource_find(devlink, NULL, resource_id);
+-	if (WARN_ON(!resource))
+-		return;
+-	WARN_ON(!resource->occ_get);
+-
+-	resource->occ_get = NULL;
+-	resource->occ_get_priv = NULL;
+-}
+-EXPORT_SYMBOL_GPL(devl_resource_occ_get_unregister);
+-
+-/**
+- *	devlink_resource_occ_get_unregister - unregister occupancy getter
+- *
+- *	@devlink: devlink
+- *	@resource_id: resource id
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_resource_occ_get_unregister(struct devlink *devlink,
+-					 u64 resource_id)
+-{
+-	devl_lock(devlink);
+-	devl_resource_occ_get_unregister(devlink, resource_id);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
+-
+-static int devlink_param_verify(const struct devlink_param *param)
+-{
+-	if (!param || !param->name || !param->supported_cmodes)
+-		return -EINVAL;
+-	if (param->generic)
+-		return devlink_param_generic_verify(param);
+-	else
+-		return devlink_param_driver_verify(param);
+-}
+-
+-/**
+- *	devlink_params_register - register configuration parameters
+- *
+- *	@devlink: devlink
+- *	@params: configuration parameters array
+- *	@params_count: number of parameters provided
+- *
+- *	Register the configuration parameters supported by the driver.
+- */
+-int devlink_params_register(struct devlink *devlink,
+-			    const struct devlink_param *params,
+-			    size_t params_count)
+-{
+-	const struct devlink_param *param = params;
+-	int i, err;
+-
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	for (i = 0; i < params_count; i++, param++) {
+-		err = devlink_param_register(devlink, param);
+-		if (err)
+-			goto rollback;
+-	}
+-	return 0;
+-
+-rollback:
+-	if (!i)
+-		return err;
+-
+-	for (param--; i > 0; i--, param--)
+-		devlink_param_unregister(devlink, param);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_params_register);
+-
+-/**
+- *	devlink_params_unregister - unregister configuration parameters
+- *	@devlink: devlink
+- *	@params: configuration parameters to unregister
+- *	@params_count: number of parameters provided
+- */
+-void devlink_params_unregister(struct devlink *devlink,
+-			       const struct devlink_param *params,
+-			       size_t params_count)
+-{
+-	const struct devlink_param *param = params;
+-	int i;
+-
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	for (i = 0; i < params_count; i++, param++)
+-		devlink_param_unregister(devlink, param);
+-}
+-EXPORT_SYMBOL_GPL(devlink_params_unregister);
+-
+-/**
+- * devlink_param_register - register one configuration parameter
+- *
+- * @devlink: devlink
+- * @param: one configuration parameter
+- *
+- * Register the configuration parameter supported by the driver.
+- * Return: returns 0 on successful registration or error code otherwise.
+- */
+-int devlink_param_register(struct devlink *devlink,
+-			   const struct devlink_param *param)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	WARN_ON(devlink_param_verify(param));
+-	WARN_ON(devlink_param_find_by_name(&devlink->param_list, param->name));
+-
+-	if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
+-		WARN_ON(param->get || param->set);
+-	else
+-		WARN_ON(!param->get || !param->set);
+-
+-	param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
+-	if (!param_item)
+-		return -ENOMEM;
+-
+-	param_item->param = param;
+-
+-	list_add_tail(&param_item->list, &devlink->param_list);
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_param_register);
+-
+-/**
+- * devlink_param_unregister - unregister one configuration parameter
+- * @devlink: devlink
+- * @param: configuration parameter to unregister
+- */
+-void devlink_param_unregister(struct devlink *devlink,
+-			      const struct devlink_param *param)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	param_item =
+-		devlink_param_find_by_name(&devlink->param_list, param->name);
+-	WARN_ON(!param_item);
+-	list_del(&param_item->list);
+-	kfree(param_item);
+-}
+-EXPORT_SYMBOL_GPL(devlink_param_unregister);
+-
+-/**
+- *	devlink_param_driverinit_value_get - get configuration parameter
+- *					     value for driver initializing
+- *
+- *	@devlink: devlink
+- *	@param_id: parameter ID
+- *	@init_val: value of parameter in driverinit configuration mode
+- *
+- *	This function should be used by the driver to get driverinit
+- *	configuration for initialization after reload command.
+- */
+-int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
+-				       union devlink_param_value *init_val)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	if (!devlink_reload_supported(devlink->ops))
+-		return -EOPNOTSUPP;
+-
+-	param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
+-	if (!param_item)
+-		return -EINVAL;
+-
+-	if (!param_item->driverinit_value_valid ||
+-	    !devlink_param_cmode_is_supported(param_item->param,
+-					      DEVLINK_PARAM_CMODE_DRIVERINIT))
+-		return -EOPNOTSUPP;
+-
+-	if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
+-		strcpy(init_val->vstr, param_item->driverinit_value.vstr);
+-	else
+-		*init_val = param_item->driverinit_value;
+-
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
+-
+-/**
+- *	devlink_param_driverinit_value_set - set value of configuration
+- *					     parameter for driverinit
+- *					     configuration mode
+- *
+- *	@devlink: devlink
+- *	@param_id: parameter ID
+- *	@init_val: value of parameter to set for driverinit configuration mode
+- *
+- *	This function should be used by the driver to set driverinit
+- *	configuration mode default value.
+- */
+-int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
+-				       union devlink_param_value init_val)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
+-
+-	param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
+-	if (!param_item)
+-		return -EINVAL;
+-
+-	if (!devlink_param_cmode_is_supported(param_item->param,
+-					      DEVLINK_PARAM_CMODE_DRIVERINIT))
+-		return -EOPNOTSUPP;
+-
+-	if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
+-		strcpy(param_item->driverinit_value.vstr, init_val.vstr);
+-	else
+-		param_item->driverinit_value = init_val;
+-	param_item->driverinit_value_valid = true;
+-	return 0;
+-}
+-EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
+-
+-/**
+- *	devlink_param_value_changed - notify devlink on a parameter's value
+- *				      change. Should be called by the driver
+- *				      right after the change.
+- *
+- *	@devlink: devlink
+- *	@param_id: parameter ID
+- *
+- *	This function should be used by the driver to notify devlink on value
+- *	change, excluding driverinit configuration mode.
+- *	For driverinit configuration mode driver should use the function
+- */
+-void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
+-{
+-	struct devlink_param_item *param_item;
+-
+-	param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
+-	WARN_ON(!param_item);
+-
+-	devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
+-}
+-EXPORT_SYMBOL_GPL(devlink_param_value_changed);
+-
+-/**
+- * devl_region_create - create a new address region
+- *
+- * @devlink: devlink
+- * @ops: region operations and name
+- * @region_max_snapshots: Maximum supported number of snapshots for region
+- * @region_size: size of region
+- */
+-struct devlink_region *devl_region_create(struct devlink *devlink,
+-					  const struct devlink_region_ops *ops,
+-					  u32 region_max_snapshots,
+-					  u64 region_size)
+-{
+-	struct devlink_region *region;
+-
+-	devl_assert_locked(devlink);
+-
+-	if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
+-		return ERR_PTR(-EINVAL);
+-
+-	if (devlink_region_get_by_name(devlink, ops->name))
+-		return ERR_PTR(-EEXIST);
+-
+-	region = kzalloc(sizeof(*region), GFP_KERNEL);
+-	if (!region)
+-		return ERR_PTR(-ENOMEM);
+-
+-	region->devlink = devlink;
+-	region->max_snapshots = region_max_snapshots;
+-	region->ops = ops;
+-	region->size = region_size;
+-	INIT_LIST_HEAD(&region->snapshot_list);
+-	mutex_init(&region->snapshot_lock);
+-	list_add_tail(&region->list, &devlink->region_list);
+-	devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
+-
+-	return region;
+-}
+-EXPORT_SYMBOL_GPL(devl_region_create);
+-
+-/**
+- *	devlink_region_create - create a new address region
+- *
+- *	@devlink: devlink
+- *	@ops: region operations and name
+- *	@region_max_snapshots: Maximum supported number of snapshots for region
+- *	@region_size: size of region
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-struct devlink_region *
+-devlink_region_create(struct devlink *devlink,
+-		      const struct devlink_region_ops *ops,
+-		      u32 region_max_snapshots, u64 region_size)
+-{
+-	struct devlink_region *region;
+-
+-	devl_lock(devlink);
+-	region = devl_region_create(devlink, ops, region_max_snapshots,
+-				    region_size);
+-	devl_unlock(devlink);
+-	return region;
+-}
+-EXPORT_SYMBOL_GPL(devlink_region_create);
+-
+-/**
+- *	devlink_port_region_create - create a new address region for a port
+- *
+- *	@port: devlink port
+- *	@ops: region operations and name
+- *	@region_max_snapshots: Maximum supported number of snapshots for region
+- *	@region_size: size of region
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-struct devlink_region *
+-devlink_port_region_create(struct devlink_port *port,
+-			   const struct devlink_port_region_ops *ops,
+-			   u32 region_max_snapshots, u64 region_size)
+-{
+-	struct devlink *devlink = port->devlink;
+-	struct devlink_region *region;
+-	int err = 0;
+-
+-	ASSERT_DEVLINK_PORT_INITIALIZED(port);
+-
+-	if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
+-		return ERR_PTR(-EINVAL);
+-
+-	devl_lock(devlink);
+-
+-	if (devlink_port_region_get_by_name(port, ops->name)) {
+-		err = -EEXIST;
+-		goto unlock;
+-	}
+-
+-	region = kzalloc(sizeof(*region), GFP_KERNEL);
+-	if (!region) {
+-		err = -ENOMEM;
+-		goto unlock;
+-	}
+-
+-	region->devlink = devlink;
+-	region->port = port;
+-	region->max_snapshots = region_max_snapshots;
+-	region->port_ops = ops;
+-	region->size = region_size;
+-	INIT_LIST_HEAD(&region->snapshot_list);
+-	mutex_init(&region->snapshot_lock);
+-	list_add_tail(&region->list, &port->region_list);
+-	devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
+-
+-	devl_unlock(devlink);
+-	return region;
+-
+-unlock:
+-	devl_unlock(devlink);
+-	return ERR_PTR(err);
+-}
+-EXPORT_SYMBOL_GPL(devlink_port_region_create);
+-
+-/**
+- * devl_region_destroy - destroy address region
+- *
+- * @region: devlink region to destroy
+- */
+-void devl_region_destroy(struct devlink_region *region)
+-{
+-	struct devlink *devlink = region->devlink;
+-	struct devlink_snapshot *snapshot, *ts;
+-
+-	devl_assert_locked(devlink);
+-
+-	/* Free all snapshots of region */
+-	mutex_lock(&region->snapshot_lock);
+-	list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
+-		devlink_region_snapshot_del(region, snapshot);
+-	mutex_unlock(&region->snapshot_lock);
+-
+-	list_del(&region->list);
+-	mutex_destroy(&region->snapshot_lock);
+-
+-	devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
+-	kfree(region);
+-}
+-EXPORT_SYMBOL_GPL(devl_region_destroy);
+-
+-/**
+- *	devlink_region_destroy - destroy address region
+- *
+- *	@region: devlink region to destroy
+- *
+- *	Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_region_destroy(struct devlink_region *region)
+-{
+-	struct devlink *devlink = region->devlink;
+-
+-	devl_lock(devlink);
+-	devl_region_destroy(region);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_region_destroy);
+-
+-/**
+- *	devlink_region_snapshot_id_get - get snapshot ID
+- *
+- *	This callback should be called when adding a new snapshot,
+- *	Driver should use the same id for multiple snapshots taken
+- *	on multiple regions at the same time/by the same trigger.
+- *
+- *	The caller of this function must use devlink_region_snapshot_id_put
+- *	when finished creating regions using this id.
+- *
+- *	Returns zero on success, or a negative error code on failure.
+- *
+- *	@devlink: devlink
+- *	@id: storage to return id
+- */
+-int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
+-{
+-	return __devlink_region_snapshot_id_get(devlink, id);
+-}
+-EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
+-
+-/**
+- *	devlink_region_snapshot_id_put - put snapshot ID reference
+- *
+- *	This should be called by a driver after finishing creating snapshots
+- *	with an id. Doing so ensures that the ID can later be released in the
+- *	event that all snapshots using it have been destroyed.
+- *
+- *	@devlink: devlink
+- *	@id: id to release reference on
+- */
+-void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
+-{
+-	__devlink_snapshot_id_decrement(devlink, id);
+-}
+-EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
+-
+-/**
+- *	devlink_region_snapshot_create - create a new snapshot
+- *	This will add a new snapshot of a region. The snapshot
+- *	will be stored on the region struct and can be accessed
+- *	from devlink. This is useful for future analyses of snapshots.
+- *	Multiple snapshots can be created on a region.
+- *	The @snapshot_id should be obtained using the getter function.
+- *
+- *	@region: devlink region of the snapshot
+- *	@data: snapshot data
+- *	@snapshot_id: snapshot id to be created
+- */
+-int devlink_region_snapshot_create(struct devlink_region *region,
+-				   u8 *data, u32 snapshot_id)
+-{
+-	int err;
+-
+-	mutex_lock(&region->snapshot_lock);
+-	err = __devlink_region_snapshot_create(region, data, snapshot_id);
+-	mutex_unlock(&region->snapshot_lock);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
+-
+-#define DEVLINK_TRAP(_id, _type)					      \
+-	{								      \
+-		.type = DEVLINK_TRAP_TYPE_##_type,			      \
+-		.id = DEVLINK_TRAP_GENERIC_ID_##_id,			      \
+-		.name = DEVLINK_TRAP_GENERIC_NAME_##_id,		      \
+-	}
+-
+-static const struct devlink_trap devlink_trap_generic[] = {
+-	DEVLINK_TRAP(SMAC_MC, DROP),
+-	DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
+-	DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
+-	DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
+-	DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
+-	DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
+-	DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
+-	DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
+-	DEVLINK_TRAP(TAIL_DROP, DROP),
+-	DEVLINK_TRAP(NON_IP_PACKET, DROP),
+-	DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
+-	DEVLINK_TRAP(DIP_LB, DROP),
+-	DEVLINK_TRAP(SIP_MC, DROP),
+-	DEVLINK_TRAP(SIP_LB, DROP),
+-	DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
+-	DEVLINK_TRAP(IPV4_SIP_BC, DROP),
+-	DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
+-	DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
+-	DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
+-	DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
+-	DEVLINK_TRAP(RPF, EXCEPTION),
+-	DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
+-	DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
+-	DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
+-	DEVLINK_TRAP(NON_ROUTABLE, DROP),
+-	DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
+-	DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
+-	DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
+-	DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
+-	DEVLINK_TRAP(STP, CONTROL),
+-	DEVLINK_TRAP(LACP, CONTROL),
+-	DEVLINK_TRAP(LLDP, CONTROL),
+-	DEVLINK_TRAP(IGMP_QUERY, CONTROL),
+-	DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
+-	DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
+-	DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
+-	DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
+-	DEVLINK_TRAP(MLD_QUERY, CONTROL),
+-	DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
+-	DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
+-	DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
+-	DEVLINK_TRAP(IPV4_DHCP, CONTROL),
+-	DEVLINK_TRAP(IPV6_DHCP, CONTROL),
+-	DEVLINK_TRAP(ARP_REQUEST, CONTROL),
+-	DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
+-	DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
+-	DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
+-	DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
+-	DEVLINK_TRAP(IPV4_BFD, CONTROL),
+-	DEVLINK_TRAP(IPV6_BFD, CONTROL),
+-	DEVLINK_TRAP(IPV4_OSPF, CONTROL),
+-	DEVLINK_TRAP(IPV6_OSPF, CONTROL),
+-	DEVLINK_TRAP(IPV4_BGP, CONTROL),
+-	DEVLINK_TRAP(IPV6_BGP, CONTROL),
+-	DEVLINK_TRAP(IPV4_VRRP, CONTROL),
+-	DEVLINK_TRAP(IPV6_VRRP, CONTROL),
+-	DEVLINK_TRAP(IPV4_PIM, CONTROL),
+-	DEVLINK_TRAP(IPV6_PIM, CONTROL),
+-	DEVLINK_TRAP(UC_LB, CONTROL),
+-	DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
+-	DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
+-	DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
+-	DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
+-	DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
+-	DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
+-	DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
+-	DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
+-	DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
+-	DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
+-	DEVLINK_TRAP(PTP_EVENT, CONTROL),
+-	DEVLINK_TRAP(PTP_GENERAL, CONTROL),
+-	DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
+-	DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
+-	DEVLINK_TRAP(EARLY_DROP, DROP),
+-	DEVLINK_TRAP(VXLAN_PARSING, DROP),
+-	DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
+-	DEVLINK_TRAP(VLAN_PARSING, DROP),
+-	DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
+-	DEVLINK_TRAP(MPLS_PARSING, DROP),
+-	DEVLINK_TRAP(ARP_PARSING, DROP),
+-	DEVLINK_TRAP(IP_1_PARSING, DROP),
+-	DEVLINK_TRAP(IP_N_PARSING, DROP),
+-	DEVLINK_TRAP(GRE_PARSING, DROP),
+-	DEVLINK_TRAP(UDP_PARSING, DROP),
+-	DEVLINK_TRAP(TCP_PARSING, DROP),
+-	DEVLINK_TRAP(IPSEC_PARSING, DROP),
+-	DEVLINK_TRAP(SCTP_PARSING, DROP),
+-	DEVLINK_TRAP(DCCP_PARSING, DROP),
+-	DEVLINK_TRAP(GTP_PARSING, DROP),
+-	DEVLINK_TRAP(ESP_PARSING, DROP),
+-	DEVLINK_TRAP(BLACKHOLE_NEXTHOP, DROP),
+-	DEVLINK_TRAP(DMAC_FILTER, DROP),
+-};
+-
+-#define DEVLINK_TRAP_GROUP(_id)						      \
+-	{								      \
+-		.id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,		      \
+-		.name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,		      \
+-	}
+-
+-static const struct devlink_trap_group devlink_trap_group_generic[] = {
+-	DEVLINK_TRAP_GROUP(L2_DROPS),
+-	DEVLINK_TRAP_GROUP(L3_DROPS),
+-	DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
+-	DEVLINK_TRAP_GROUP(BUFFER_DROPS),
+-	DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
+-	DEVLINK_TRAP_GROUP(ACL_DROPS),
+-	DEVLINK_TRAP_GROUP(STP),
+-	DEVLINK_TRAP_GROUP(LACP),
+-	DEVLINK_TRAP_GROUP(LLDP),
+-	DEVLINK_TRAP_GROUP(MC_SNOOPING),
+-	DEVLINK_TRAP_GROUP(DHCP),
+-	DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
+-	DEVLINK_TRAP_GROUP(BFD),
+-	DEVLINK_TRAP_GROUP(OSPF),
+-	DEVLINK_TRAP_GROUP(BGP),
+-	DEVLINK_TRAP_GROUP(VRRP),
+-	DEVLINK_TRAP_GROUP(PIM),
+-	DEVLINK_TRAP_GROUP(UC_LB),
+-	DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
+-	DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
+-	DEVLINK_TRAP_GROUP(IPV6),
+-	DEVLINK_TRAP_GROUP(PTP_EVENT),
+-	DEVLINK_TRAP_GROUP(PTP_GENERAL),
+-	DEVLINK_TRAP_GROUP(ACL_SAMPLE),
+-	DEVLINK_TRAP_GROUP(ACL_TRAP),
+-	DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
+-};
+-
+-static int devlink_trap_generic_verify(const struct devlink_trap *trap)
+-{
+-	if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
+-		return -EINVAL;
+-
+-	if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
+-		return -EINVAL;
+-
+-	if (trap->type != devlink_trap_generic[trap->id].type)
+-		return -EINVAL;
+-
+-	return 0;
+-}
+-
+-static int devlink_trap_driver_verify(const struct devlink_trap *trap)
+-{
+-	int i;
+-
+-	if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
+-		return -EINVAL;
+-
+-	for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
+-		if (!strcmp(trap->name, devlink_trap_generic[i].name))
+-			return -EEXIST;
+-	}
+-
+-	return 0;
+-}
+-
+-static int devlink_trap_verify(const struct devlink_trap *trap)
+-{
+-	if (!trap || !trap->name)
+-		return -EINVAL;
+-
+-	if (trap->generic)
+-		return devlink_trap_generic_verify(trap);
+-	else
+-		return devlink_trap_driver_verify(trap);
+-}
+-
+-static int
+-devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
+-{
+-	if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
+-		return -EINVAL;
+-
+-	if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
+-		return -EINVAL;
+-
+-	return 0;
+-}
+-
+-static int
+-devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
+-{
+-	int i;
+-
+-	if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
+-		return -EINVAL;
+-
+-	for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
+-		if (!strcmp(group->name, devlink_trap_group_generic[i].name))
+-			return -EEXIST;
+-	}
+-
+-	return 0;
+-}
+-
+-static int devlink_trap_group_verify(const struct devlink_trap_group *group)
+-{
+-	if (group->generic)
+-		return devlink_trap_group_generic_verify(group);
+-	else
+-		return devlink_trap_group_driver_verify(group);
+-}
+-
+-static void
+-devlink_trap_group_notify(struct devlink *devlink,
+-			  const struct devlink_trap_group_item *group_item,
+-			  enum devlink_command cmd)
+-{
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
+-		     cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
+-					 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int
+-devlink_trap_item_group_link(struct devlink *devlink,
+-			     struct devlink_trap_item *trap_item)
+-{
+-	u16 group_id = trap_item->trap->init_group_id;
+-	struct devlink_trap_group_item *group_item;
+-
+-	group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
+-	if (WARN_ON_ONCE(!group_item))
+-		return -EINVAL;
+-
+-	trap_item->group_item = group_item;
+-
+-	return 0;
+-}
+-
+-static void devlink_trap_notify(struct devlink *devlink,
+-				const struct devlink_trap_item *trap_item,
+-				enum devlink_command cmd)
+-{
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
+-		     cmd != DEVLINK_CMD_TRAP_DEL);
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int
+-devlink_trap_register(struct devlink *devlink,
+-		      const struct devlink_trap *trap, void *priv)
+-{
+-	struct devlink_trap_item *trap_item;
+-	int err;
+-
+-	if (devlink_trap_item_lookup(devlink, trap->name))
+-		return -EEXIST;
+-
+-	trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
+-	if (!trap_item)
+-		return -ENOMEM;
+-
+-	trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
+-	if (!trap_item->stats) {
+-		err = -ENOMEM;
+-		goto err_stats_alloc;
+-	}
+-
+-	trap_item->trap = trap;
+-	trap_item->action = trap->init_action;
+-	trap_item->priv = priv;
+-
+-	err = devlink_trap_item_group_link(devlink, trap_item);
+-	if (err)
+-		goto err_group_link;
+-
+-	err = devlink->ops->trap_init(devlink, trap, trap_item);
+-	if (err)
+-		goto err_trap_init;
+-
+-	list_add_tail(&trap_item->list, &devlink->trap_list);
+-	devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
+-
+-	return 0;
+-
+-err_trap_init:
+-err_group_link:
+-	free_percpu(trap_item->stats);
+-err_stats_alloc:
+-	kfree(trap_item);
+-	return err;
+-}
+-
+-static void devlink_trap_unregister(struct devlink *devlink,
+-				    const struct devlink_trap *trap)
+-{
+-	struct devlink_trap_item *trap_item;
+-
+-	trap_item = devlink_trap_item_lookup(devlink, trap->name);
+-	if (WARN_ON_ONCE(!trap_item))
+-		return;
+-
+-	devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
+-	list_del(&trap_item->list);
+-	if (devlink->ops->trap_fini)
+-		devlink->ops->trap_fini(devlink, trap, trap_item);
+-	free_percpu(trap_item->stats);
+-	kfree(trap_item);
+-}
+-
+-static void devlink_trap_disable(struct devlink *devlink,
+-				 const struct devlink_trap *trap)
+-{
+-	struct devlink_trap_item *trap_item;
+-
+-	trap_item = devlink_trap_item_lookup(devlink, trap->name);
+-	if (WARN_ON_ONCE(!trap_item))
+-		return;
+-
+-	devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
+-				      NULL);
+-	trap_item->action = DEVLINK_TRAP_ACTION_DROP;
+-}
+-
+-/**
+- * devl_traps_register - Register packet traps with devlink.
+- * @devlink: devlink.
+- * @traps: Packet traps.
+- * @traps_count: Count of provided packet traps.
+- * @priv: Driver private information.
+- *
+- * Return: Non-zero value on failure.
+- */
+-int devl_traps_register(struct devlink *devlink,
+-			const struct devlink_trap *traps,
+-			size_t traps_count, void *priv)
+-{
+-	int i, err;
+-
+-	if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
+-		return -EINVAL;
+-
+-	devl_assert_locked(devlink);
+-	for (i = 0; i < traps_count; i++) {
+-		const struct devlink_trap *trap = &traps[i];
+-
+-		err = devlink_trap_verify(trap);
+-		if (err)
+-			goto err_trap_verify;
+-
+-		err = devlink_trap_register(devlink, trap, priv);
+-		if (err)
+-			goto err_trap_register;
+-	}
+-
+-	return 0;
+-
+-err_trap_register:
+-err_trap_verify:
+-	for (i--; i >= 0; i--)
+-		devlink_trap_unregister(devlink, &traps[i]);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devl_traps_register);
+-
+-/**
+- * devlink_traps_register - Register packet traps with devlink.
+- * @devlink: devlink.
+- * @traps: Packet traps.
+- * @traps_count: Count of provided packet traps.
+- * @priv: Driver private information.
+- *
+- * Context: Takes and release devlink->lock <mutex>.
+- *
+- * Return: Non-zero value on failure.
+- */
+-int devlink_traps_register(struct devlink *devlink,
+-			   const struct devlink_trap *traps,
+-			   size_t traps_count, void *priv)
+-{
+-	int err;
+-
+-	devl_lock(devlink);
+-	err = devl_traps_register(devlink, traps, traps_count, priv);
+-	devl_unlock(devlink);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_traps_register);
+-
+-/**
+- * devl_traps_unregister - Unregister packet traps from devlink.
+- * @devlink: devlink.
+- * @traps: Packet traps.
+- * @traps_count: Count of provided packet traps.
+- */
+-void devl_traps_unregister(struct devlink *devlink,
+-			   const struct devlink_trap *traps,
+-			   size_t traps_count)
+-{
+-	int i;
+-
+-	devl_assert_locked(devlink);
+-	/* Make sure we do not have any packets in-flight while unregistering
+-	 * traps by disabling all of them and waiting for a grace period.
+-	 */
+-	for (i = traps_count - 1; i >= 0; i--)
+-		devlink_trap_disable(devlink, &traps[i]);
+-	synchronize_rcu();
+-	for (i = traps_count - 1; i >= 0; i--)
+-		devlink_trap_unregister(devlink, &traps[i]);
+-}
+-EXPORT_SYMBOL_GPL(devl_traps_unregister);
+-
+-/**
+- * devlink_traps_unregister - Unregister packet traps from devlink.
+- * @devlink: devlink.
+- * @traps: Packet traps.
+- * @traps_count: Count of provided packet traps.
+- *
+- * Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_traps_unregister(struct devlink *devlink,
+-			      const struct devlink_trap *traps,
+-			      size_t traps_count)
+-{
+-	devl_lock(devlink);
+-	devl_traps_unregister(devlink, traps, traps_count);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_traps_unregister);
+-
+-static void
+-devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
+-			  size_t skb_len)
+-{
+-	struct devlink_stats *stats;
+-
+-	stats = this_cpu_ptr(trap_stats);
+-	u64_stats_update_begin(&stats->syncp);
+-	u64_stats_add(&stats->rx_bytes, skb_len);
+-	u64_stats_inc(&stats->rx_packets);
+-	u64_stats_update_end(&stats->syncp);
+-}
+-
+-static void
+-devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
+-				 const struct devlink_trap_item *trap_item,
+-				 struct devlink_port *in_devlink_port,
+-				 const struct flow_action_cookie *fa_cookie)
+-{
+-	metadata->trap_name = trap_item->trap->name;
+-	metadata->trap_group_name = trap_item->group_item->group->name;
+-	metadata->fa_cookie = fa_cookie;
+-	metadata->trap_type = trap_item->trap->type;
+-
+-	spin_lock(&in_devlink_port->type_lock);
+-	if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
+-		metadata->input_dev = in_devlink_port->type_dev;
+-	spin_unlock(&in_devlink_port->type_lock);
+-}
+-
+-/**
+- * devlink_trap_report - Report trapped packet to drop monitor.
+- * @devlink: devlink.
+- * @skb: Trapped packet.
+- * @trap_ctx: Trap context.
+- * @in_devlink_port: Input devlink port.
+- * @fa_cookie: Flow action cookie. Could be NULL.
+- */
+-void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
+-			 void *trap_ctx, struct devlink_port *in_devlink_port,
+-			 const struct flow_action_cookie *fa_cookie)
+-
+-{
+-	struct devlink_trap_item *trap_item = trap_ctx;
+-
+-	devlink_trap_stats_update(trap_item->stats, skb->len);
+-	devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
+-
+-	if (trace_devlink_trap_report_enabled()) {
+-		struct devlink_trap_metadata metadata = {};
+-
+-		devlink_trap_report_metadata_set(&metadata, trap_item,
+-						 in_devlink_port, fa_cookie);
+-		trace_devlink_trap_report(devlink, skb, &metadata);
+-	}
+-}
+-EXPORT_SYMBOL_GPL(devlink_trap_report);
+-
+-/**
+- * devlink_trap_ctx_priv - Trap context to driver private information.
+- * @trap_ctx: Trap context.
+- *
+- * Return: Driver private information passed during registration.
+- */
+-void *devlink_trap_ctx_priv(void *trap_ctx)
+-{
+-	struct devlink_trap_item *trap_item = trap_ctx;
+-
+-	return trap_item->priv;
+-}
+-EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
+-
+-static int
+-devlink_trap_group_item_policer_link(struct devlink *devlink,
+-				     struct devlink_trap_group_item *group_item)
+-{
+-	u32 policer_id = group_item->group->init_policer_id;
+-	struct devlink_trap_policer_item *policer_item;
+-
+-	if (policer_id == 0)
+-		return 0;
+-
+-	policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
+-	if (WARN_ON_ONCE(!policer_item))
+-		return -EINVAL;
+-
+-	group_item->policer_item = policer_item;
+-
+-	return 0;
+-}
+-
+-static int
+-devlink_trap_group_register(struct devlink *devlink,
+-			    const struct devlink_trap_group *group)
+-{
+-	struct devlink_trap_group_item *group_item;
+-	int err;
+-
+-	if (devlink_trap_group_item_lookup(devlink, group->name))
+-		return -EEXIST;
+-
+-	group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
+-	if (!group_item)
+-		return -ENOMEM;
+-
+-	group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
+-	if (!group_item->stats) {
+-		err = -ENOMEM;
+-		goto err_stats_alloc;
+-	}
+-
+-	group_item->group = group;
+-
+-	err = devlink_trap_group_item_policer_link(devlink, group_item);
+-	if (err)
+-		goto err_policer_link;
+-
+-	if (devlink->ops->trap_group_init) {
+-		err = devlink->ops->trap_group_init(devlink, group);
+-		if (err)
+-			goto err_group_init;
+-	}
+-
+-	list_add_tail(&group_item->list, &devlink->trap_group_list);
+-	devlink_trap_group_notify(devlink, group_item,
+-				  DEVLINK_CMD_TRAP_GROUP_NEW);
+-
+-	return 0;
+-
+-err_group_init:
+-err_policer_link:
+-	free_percpu(group_item->stats);
+-err_stats_alloc:
+-	kfree(group_item);
+-	return err;
+-}
+-
+-static void
+-devlink_trap_group_unregister(struct devlink *devlink,
+-			      const struct devlink_trap_group *group)
+-{
+-	struct devlink_trap_group_item *group_item;
+-
+-	group_item = devlink_trap_group_item_lookup(devlink, group->name);
+-	if (WARN_ON_ONCE(!group_item))
+-		return;
+-
+-	devlink_trap_group_notify(devlink, group_item,
+-				  DEVLINK_CMD_TRAP_GROUP_DEL);
+-	list_del(&group_item->list);
+-	free_percpu(group_item->stats);
+-	kfree(group_item);
+-}
+-
+-/**
+- * devl_trap_groups_register - Register packet trap groups with devlink.
+- * @devlink: devlink.
+- * @groups: Packet trap groups.
+- * @groups_count: Count of provided packet trap groups.
+- *
+- * Return: Non-zero value on failure.
+- */
+-int devl_trap_groups_register(struct devlink *devlink,
+-			      const struct devlink_trap_group *groups,
+-			      size_t groups_count)
+-{
+-	int i, err;
+-
+-	devl_assert_locked(devlink);
+-	for (i = 0; i < groups_count; i++) {
+-		const struct devlink_trap_group *group = &groups[i];
+-
+-		err = devlink_trap_group_verify(group);
+-		if (err)
+-			goto err_trap_group_verify;
+-
+-		err = devlink_trap_group_register(devlink, group);
+-		if (err)
+-			goto err_trap_group_register;
+-	}
+-
+-	return 0;
+-
+-err_trap_group_register:
+-err_trap_group_verify:
+-	for (i--; i >= 0; i--)
+-		devlink_trap_group_unregister(devlink, &groups[i]);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devl_trap_groups_register);
+-
+-/**
+- * devlink_trap_groups_register - Register packet trap groups with devlink.
+- * @devlink: devlink.
+- * @groups: Packet trap groups.
+- * @groups_count: Count of provided packet trap groups.
+- *
+- * Context: Takes and release devlink->lock <mutex>.
+- *
+- * Return: Non-zero value on failure.
+- */
+-int devlink_trap_groups_register(struct devlink *devlink,
+-				 const struct devlink_trap_group *groups,
+-				 size_t groups_count)
+-{
+-	int err;
+-
+-	devl_lock(devlink);
+-	err = devl_trap_groups_register(devlink, groups, groups_count);
+-	devl_unlock(devlink);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
+-
+-/**
+- * devl_trap_groups_unregister - Unregister packet trap groups from devlink.
+- * @devlink: devlink.
+- * @groups: Packet trap groups.
+- * @groups_count: Count of provided packet trap groups.
+- */
+-void devl_trap_groups_unregister(struct devlink *devlink,
+-				 const struct devlink_trap_group *groups,
+-				 size_t groups_count)
+-{
+-	int i;
+-
+-	devl_assert_locked(devlink);
+-	for (i = groups_count - 1; i >= 0; i--)
+-		devlink_trap_group_unregister(devlink, &groups[i]);
+-}
+-EXPORT_SYMBOL_GPL(devl_trap_groups_unregister);
+-
+-/**
+- * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
+- * @devlink: devlink.
+- * @groups: Packet trap groups.
+- * @groups_count: Count of provided packet trap groups.
+- *
+- * Context: Takes and release devlink->lock <mutex>.
+- */
+-void devlink_trap_groups_unregister(struct devlink *devlink,
+-				    const struct devlink_trap_group *groups,
+-				    size_t groups_count)
+-{
+-	devl_lock(devlink);
+-	devl_trap_groups_unregister(devlink, groups, groups_count);
+-	devl_unlock(devlink);
+-}
+-EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
+-
+-static void
+-devlink_trap_policer_notify(struct devlink *devlink,
+-			    const struct devlink_trap_policer_item *policer_item,
+-			    enum devlink_command cmd)
+-{
+-	struct sk_buff *msg;
+-	int err;
+-
+-	WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
+-		     cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
+-	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
+-		return;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
+-					   0, 0);
+-	if (err) {
+-		nlmsg_free(msg);
+-		return;
+-	}
+-
+-	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
+-				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
+-}
+-
+-static int
+-devlink_trap_policer_register(struct devlink *devlink,
+-			      const struct devlink_trap_policer *policer)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-	int err;
+-
+-	if (devlink_trap_policer_item_lookup(devlink, policer->id))
+-		return -EEXIST;
+-
+-	policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
+-	if (!policer_item)
+-		return -ENOMEM;
+-
+-	policer_item->policer = policer;
+-	policer_item->rate = policer->init_rate;
+-	policer_item->burst = policer->init_burst;
+-
+-	if (devlink->ops->trap_policer_init) {
+-		err = devlink->ops->trap_policer_init(devlink, policer);
+-		if (err)
+-			goto err_policer_init;
+-	}
+-
+-	list_add_tail(&policer_item->list, &devlink->trap_policer_list);
+-	devlink_trap_policer_notify(devlink, policer_item,
+-				    DEVLINK_CMD_TRAP_POLICER_NEW);
+-
+-	return 0;
+-
+-err_policer_init:
+-	kfree(policer_item);
+-	return err;
+-}
+-
+-static void
+-devlink_trap_policer_unregister(struct devlink *devlink,
+-				const struct devlink_trap_policer *policer)
+-{
+-	struct devlink_trap_policer_item *policer_item;
+-
+-	policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
+-	if (WARN_ON_ONCE(!policer_item))
+-		return;
+-
+-	devlink_trap_policer_notify(devlink, policer_item,
+-				    DEVLINK_CMD_TRAP_POLICER_DEL);
+-	list_del(&policer_item->list);
+-	if (devlink->ops->trap_policer_fini)
+-		devlink->ops->trap_policer_fini(devlink, policer);
+-	kfree(policer_item);
+-}
+-
+-/**
+- * devl_trap_policers_register - Register packet trap policers with devlink.
+- * @devlink: devlink.
+- * @policers: Packet trap policers.
+- * @policers_count: Count of provided packet trap policers.
+- *
+- * Return: Non-zero value on failure.
+- */
+-int
+-devl_trap_policers_register(struct devlink *devlink,
+-			    const struct devlink_trap_policer *policers,
+-			    size_t policers_count)
+-{
+-	int i, err;
+-
+-	devl_assert_locked(devlink);
+-	for (i = 0; i < policers_count; i++) {
+-		const struct devlink_trap_policer *policer = &policers[i];
+-
+-		if (WARN_ON(policer->id == 0 ||
+-			    policer->max_rate < policer->min_rate ||
+-			    policer->max_burst < policer->min_burst)) {
+-			err = -EINVAL;
+-			goto err_trap_policer_verify;
+-		}
+-
+-		err = devlink_trap_policer_register(devlink, policer);
+-		if (err)
+-			goto err_trap_policer_register;
+-	}
+-	return 0;
+-
+-err_trap_policer_register:
+-err_trap_policer_verify:
+-	for (i--; i >= 0; i--)
+-		devlink_trap_policer_unregister(devlink, &policers[i]);
+-	return err;
+-}
+-EXPORT_SYMBOL_GPL(devl_trap_policers_register);
+-
+-/**
+- * devl_trap_policers_unregister - Unregister packet trap policers from devlink.
+- * @devlink: devlink.
+- * @policers: Packet trap policers.
+- * @policers_count: Count of provided packet trap policers.
+- */
+-void
+-devl_trap_policers_unregister(struct devlink *devlink,
+-			      const struct devlink_trap_policer *policers,
+-			      size_t policers_count)
+-{
+-	int i;
+-
+-	devl_assert_locked(devlink);
+-	for (i = policers_count - 1; i >= 0; i--)
+-		devlink_trap_policer_unregister(devlink, &policers[i]);
+-}
+-EXPORT_SYMBOL_GPL(devl_trap_policers_unregister);
+-
+-static void __devlink_compat_running_version(struct devlink *devlink,
+-					     char *buf, size_t len)
+-{
+-	struct devlink_info_req req = {};
+-	const struct nlattr *nlattr;
+-	struct sk_buff *msg;
+-	int rem, err;
+-
+-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+-	if (!msg)
+-		return;
+-
+-	req.msg = msg;
+-	err = devlink->ops->info_get(devlink, &req, NULL);
+-	if (err)
+-		goto free_msg;
+-
+-	nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
+-		const struct nlattr *kv;
+-		int rem_kv;
+-
+-		if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
+-			continue;
+-
+-		nla_for_each_nested(kv, nlattr, rem_kv) {
+-			if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
+-				continue;
+-
+-			strlcat(buf, nla_data(kv), len);
+-			strlcat(buf, " ", len);
+-		}
+-	}
+-free_msg:
+-	nlmsg_free(msg);
+-}
+-
+-static struct devlink_port *netdev_to_devlink_port(struct net_device *dev)
+-{
+-	if (!dev->netdev_ops->ndo_get_devlink_port)
+-		return NULL;
+-
+-	return dev->netdev_ops->ndo_get_devlink_port(dev);
+-}
+-
+-void devlink_compat_running_version(struct devlink *devlink,
+-				    char *buf, size_t len)
+-{
+-	if (!devlink->ops->info_get)
+-		return;
+-
+-	devl_lock(devlink);
+-	__devlink_compat_running_version(devlink, buf, len);
+-	devl_unlock(devlink);
+-}
+-
+-int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
+-{
+-	struct devlink_flash_update_params params = {};
+-	int ret;
+-
+-	if (!devlink->ops->flash_update)
+-		return -EOPNOTSUPP;
+-
+-	ret = request_firmware(&params.fw, file_name, devlink->dev);
+-	if (ret)
+-		return ret;
+-
+-	devl_lock(devlink);
+-	devlink_flash_update_begin_notify(devlink);
+-	ret = devlink->ops->flash_update(devlink, &params, NULL);
+-	devlink_flash_update_end_notify(devlink);
+-	devl_unlock(devlink);
+-
+-	release_firmware(params.fw);
+-
+-	return ret;
+-}
+-
+-int devlink_compat_phys_port_name_get(struct net_device *dev,
+-				      char *name, size_t len)
+-{
+-	struct devlink_port *devlink_port;
+-
+-	/* RTNL mutex is held here which ensures that devlink_port
+-	 * instance cannot disappear in the middle. No need to take
+-	 * any devlink lock as only permanent values are accessed.
+-	 */
+-	ASSERT_RTNL();
+-
+-	devlink_port = netdev_to_devlink_port(dev);
+-	if (!devlink_port)
+-		return -EOPNOTSUPP;
+-
+-	return __devlink_port_phys_port_name_get(devlink_port, name, len);
+-}
+-
+-int devlink_compat_switch_id_get(struct net_device *dev,
+-				 struct netdev_phys_item_id *ppid)
+-{
+-	struct devlink_port *devlink_port;
+-
+-	/* Caller must hold RTNL mutex or reference to dev, which ensures that
+-	 * devlink_port instance cannot disappear in the middle. No need to take
+-	 * any devlink lock as only permanent values are accessed.
+-	 */
+-	devlink_port = netdev_to_devlink_port(dev);
+-	if (!devlink_port || !devlink_port->switch_port)
+-		return -EOPNOTSUPP;
+-
+-	memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
+-
+-	return 0;
+-}
+-
+-static void __net_exit devlink_pernet_pre_exit(struct net *net)
+-{
+-	struct devlink *devlink;
+-	u32 actions_performed;
+-	unsigned long index;
+-	int err;
+-
+-	/* In case network namespace is getting destroyed, reload
+-	 * all devlink instances from this namespace into init_net.
+-	 */
+-	devlinks_xa_for_each_registered_get(net, index, devlink) {
+-		WARN_ON(!(devlink->features & DEVLINK_F_RELOAD));
+-		mutex_lock(&devlink->lock);
+-		err = devlink_reload(devlink, &init_net,
+-				     DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
+-				     DEVLINK_RELOAD_LIMIT_UNSPEC,
+-				     &actions_performed, NULL);
+-		mutex_unlock(&devlink->lock);
+-		if (err && err != -EOPNOTSUPP)
+-			pr_warn("Failed to reload devlink instance into init_net\n");
+-		devlink_put(devlink);
+-	}
+-}
+-
+-static struct pernet_operations devlink_pernet_ops __net_initdata = {
+-	.pre_exit = devlink_pernet_pre_exit,
+-};
+-
+-static int __init devlink_init(void)
+-{
+-	int err;
+-
+-	err = genl_register_family(&devlink_nl_family);
+-	if (err)
+-		goto out;
+-	err = register_pernet_subsys(&devlink_pernet_ops);
+-
+-out:
+-	WARN_ON(err);
+-	return err;
+-}
+-
+-subsys_initcall(devlink_init);
+diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
+index 2758b3f7c0214..9d4507aa736b7 100644
+--- a/net/core/rtnetlink.c
++++ b/net/core/rtnetlink.c
+@@ -2220,13 +2220,27 @@ out_err:
+ 	return err;
+ }
+ 
+-int rtnl_nla_parse_ifla(struct nlattr **tb, const struct nlattr *head, int len,
+-			struct netlink_ext_ack *exterr)
++int rtnl_nla_parse_ifinfomsg(struct nlattr **tb, const struct nlattr *nla_peer,
++			     struct netlink_ext_ack *exterr)
+ {
+-	return nla_parse_deprecated(tb, IFLA_MAX, head, len, ifla_policy,
++	const struct ifinfomsg *ifmp;
++	const struct nlattr *attrs;
++	size_t len;
++
++	ifmp = nla_data(nla_peer);
++	attrs = nla_data(nla_peer) + sizeof(struct ifinfomsg);
++	len = nla_len(nla_peer) - sizeof(struct ifinfomsg);
++
++	if (ifmp->ifi_index < 0) {
++		NL_SET_ERR_MSG_ATTR(exterr, nla_peer,
++				    "ifindex can't be negative");
++		return -EINVAL;
++	}
++
++	return nla_parse_deprecated(tb, IFLA_MAX, attrs, len, ifla_policy,
+ 				    exterr);
+ }
+-EXPORT_SYMBOL(rtnl_nla_parse_ifla);
++EXPORT_SYMBOL(rtnl_nla_parse_ifinfomsg);
+ 
+ struct net *rtnl_link_get_net(struct net *src_net, struct nlattr *tb[])
+ {
+@@ -3451,6 +3465,9 @@ replay:
+ 	if (ifm->ifi_index > 0) {
+ 		link_specified = true;
+ 		dev = __dev_get_by_index(net, ifm->ifi_index);
++	} else if (ifm->ifi_index < 0) {
++		NL_SET_ERR_MSG(extack, "ifindex can't be negative");
++		return -EINVAL;
+ 	} else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME]) {
+ 		link_specified = true;
+ 		dev = rtnl_dev_get(net, tb);
+diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
+index b780827f5e0a5..bfececa9e244e 100644
+--- a/net/dccp/ipv4.c
++++ b/net/dccp/ipv4.c
+@@ -130,7 +130,7 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
+ 						    inet->inet_daddr,
+ 						    inet->inet_sport,
+ 						    inet->inet_dport);
+-	inet->inet_id = get_random_u16();
++	atomic_set(&inet->inet_id, get_random_u16());
+ 
+ 	err = dccp_connect(sk);
+ 	rt = NULL;
+@@ -430,7 +430,7 @@ struct sock *dccp_v4_request_recv_sock(const struct sock *sk,
+ 	RCU_INIT_POINTER(newinet->inet_opt, rcu_dereference(ireq->ireq_opt));
+ 	newinet->mc_index  = inet_iif(skb);
+ 	newinet->mc_ttl	   = ip_hdr(skb)->ttl;
+-	newinet->inet_id   = get_random_u16();
++	atomic_set(&newinet->inet_id, get_random_u16());
+ 
+ 	if (dst == NULL && (dst = inet_csk_route_child_sock(sk, newsk, req)) == NULL)
+ 		goto put_and_exit;
+diff --git a/net/dccp/proto.c b/net/dccp/proto.c
+index abc02d25edc14..c522c76a9f89f 100644
+--- a/net/dccp/proto.c
++++ b/net/dccp/proto.c
+@@ -312,11 +312,15 @@ EXPORT_SYMBOL_GPL(dccp_disconnect);
+ __poll_t dccp_poll(struct file *file, struct socket *sock,
+ 		       poll_table *wait)
+ {
+-	__poll_t mask;
+ 	struct sock *sk = sock->sk;
++	__poll_t mask;
++	u8 shutdown;
++	int state;
+ 
+ 	sock_poll_wait(file, sock, wait);
+-	if (sk->sk_state == DCCP_LISTEN)
++
++	state = inet_sk_state_load(sk);
++	if (state == DCCP_LISTEN)
+ 		return inet_csk_listen_poll(sk);
+ 
+ 	/* Socket is not locked. We are protected from async events
+@@ -325,20 +329,21 @@ __poll_t dccp_poll(struct file *file, struct socket *sock,
+ 	 */
+ 
+ 	mask = 0;
+-	if (sk->sk_err)
++	if (READ_ONCE(sk->sk_err))
+ 		mask = EPOLLERR;
++	shutdown = READ_ONCE(sk->sk_shutdown);
+ 
+-	if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == DCCP_CLOSED)
++	if (shutdown == SHUTDOWN_MASK || state == DCCP_CLOSED)
+ 		mask |= EPOLLHUP;
+-	if (sk->sk_shutdown & RCV_SHUTDOWN)
++	if (shutdown & RCV_SHUTDOWN)
+ 		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
+ 
+ 	/* Connected? */
+-	if ((1 << sk->sk_state) & ~(DCCPF_REQUESTING | DCCPF_RESPOND)) {
++	if ((1 << state) & ~(DCCPF_REQUESTING | DCCPF_RESPOND)) {
+ 		if (atomic_read(&sk->sk_rmem_alloc) > 0)
+ 			mask |= EPOLLIN | EPOLLRDNORM;
+ 
+-		if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
++		if (!(shutdown & SEND_SHUTDOWN)) {
+ 			if (sk_stream_is_writeable(sk)) {
+ 				mask |= EPOLLOUT | EPOLLWRNORM;
+ 			} else {  /* send SIGIO later */
+@@ -356,7 +361,6 @@ __poll_t dccp_poll(struct file *file, struct socket *sock,
+ 	}
+ 	return mask;
+ }
+-
+ EXPORT_SYMBOL_GPL(dccp_poll);
+ 
+ int dccp_ioctl(struct sock *sk, int cmd, unsigned long arg)
+diff --git a/net/devlink/Makefile b/net/devlink/Makefile
+new file mode 100644
+index 0000000000000..3a60959f71eea
+--- /dev/null
++++ b/net/devlink/Makefile
+@@ -0,0 +1,3 @@
++# SPDX-License-Identifier: GPL-2.0
++
++obj-y := leftover.o
+diff --git a/net/devlink/leftover.c b/net/devlink/leftover.c
+new file mode 100644
+index 0000000000000..63188d6a50fe9
+--- /dev/null
++++ b/net/devlink/leftover.c
+@@ -0,0 +1,12550 @@
++// SPDX-License-Identifier: GPL-2.0-or-later
++/*
++ * net/core/devlink.c - Network physical/parent device Netlink interface
++ *
++ * Heavily inspired by net/wireless/
++ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
++ * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
++ */
++
++#include <linux/etherdevice.h>
++#include <linux/kernel.h>
++#include <linux/module.h>
++#include <linux/types.h>
++#include <linux/slab.h>
++#include <linux/gfp.h>
++#include <linux/device.h>
++#include <linux/list.h>
++#include <linux/netdevice.h>
++#include <linux/spinlock.h>
++#include <linux/refcount.h>
++#include <linux/workqueue.h>
++#include <linux/u64_stats_sync.h>
++#include <linux/timekeeping.h>
++#include <rdma/ib_verbs.h>
++#include <net/netlink.h>
++#include <net/genetlink.h>
++#include <net/rtnetlink.h>
++#include <net/net_namespace.h>
++#include <net/sock.h>
++#include <net/devlink.h>
++#define CREATE_TRACE_POINTS
++#include <trace/events/devlink.h>
++
++#define DEVLINK_RELOAD_STATS_ARRAY_SIZE \
++	(__DEVLINK_RELOAD_LIMIT_MAX * __DEVLINK_RELOAD_ACTION_MAX)
++
++struct devlink_dev_stats {
++	u32 reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
++	u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
++};
++
++struct devlink {
++	u32 index;
++	struct list_head port_list;
++	struct list_head rate_list;
++	struct list_head sb_list;
++	struct list_head dpipe_table_list;
++	struct list_head resource_list;
++	struct list_head param_list;
++	struct list_head region_list;
++	struct list_head reporter_list;
++	struct mutex reporters_lock; /* protects reporter_list */
++	struct devlink_dpipe_headers *dpipe_headers;
++	struct list_head trap_list;
++	struct list_head trap_group_list;
++	struct list_head trap_policer_list;
++	struct list_head linecard_list;
++	struct mutex linecards_lock; /* protects linecard_list */
++	const struct devlink_ops *ops;
++	u64 features;
++	struct xarray snapshot_ids;
++	struct devlink_dev_stats stats;
++	struct device *dev;
++	possible_net_t _net;
++	/* Serializes access to devlink instance specific objects such as
++	 * port, sb, dpipe, resource, params, region, traps and more.
++	 */
++	struct mutex lock;
++	struct lock_class_key lock_key;
++	u8 reload_failed:1;
++	refcount_t refcount;
++	struct completion comp;
++	struct rcu_head rcu;
++	char priv[] __aligned(NETDEV_ALIGN);
++};
++
++struct devlink_linecard_ops;
++struct devlink_linecard_type;
++
++struct devlink_linecard {
++	struct list_head list;
++	struct devlink *devlink;
++	unsigned int index;
++	refcount_t refcount;
++	const struct devlink_linecard_ops *ops;
++	void *priv;
++	enum devlink_linecard_state state;
++	struct mutex state_lock; /* Protects state */
++	const char *type;
++	struct devlink_linecard_type *types;
++	unsigned int types_count;
++	struct devlink *nested_devlink;
++};
++
++/**
++ * struct devlink_resource - devlink resource
++ * @name: name of the resource
++ * @id: id, per devlink instance
++ * @size: size of the resource
++ * @size_new: updated size of the resource, reload is needed
++ * @size_valid: valid in case the total size of the resource is valid
++ *              including its children
++ * @parent: parent resource
++ * @size_params: size parameters
++ * @list: parent list
++ * @resource_list: list of child resources
++ * @occ_get: occupancy getter callback
++ * @occ_get_priv: occupancy getter callback priv
++ */
++struct devlink_resource {
++	const char *name;
++	u64 id;
++	u64 size;
++	u64 size_new;
++	bool size_valid;
++	struct devlink_resource *parent;
++	struct devlink_resource_size_params size_params;
++	struct list_head list;
++	struct list_head resource_list;
++	devlink_resource_occ_get_t *occ_get;
++	void *occ_get_priv;
++};
++
++void *devlink_priv(struct devlink *devlink)
++{
++	return &devlink->priv;
++}
++EXPORT_SYMBOL_GPL(devlink_priv);
++
++struct devlink *priv_to_devlink(void *priv)
++{
++	return container_of(priv, struct devlink, priv);
++}
++EXPORT_SYMBOL_GPL(priv_to_devlink);
++
++struct device *devlink_to_dev(const struct devlink *devlink)
++{
++	return devlink->dev;
++}
++EXPORT_SYMBOL_GPL(devlink_to_dev);
++
++static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
++	{
++		.name = "destination mac",
++		.id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
++		.bitwidth = 48,
++	},
++};
++
++struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
++	.name = "ethernet",
++	.id = DEVLINK_DPIPE_HEADER_ETHERNET,
++	.fields = devlink_dpipe_fields_ethernet,
++	.fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
++	.global = true,
++};
++EXPORT_SYMBOL_GPL(devlink_dpipe_header_ethernet);
++
++static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
++	{
++		.name = "destination ip",
++		.id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
++		.bitwidth = 32,
++	},
++};
++
++struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
++	.name = "ipv4",
++	.id = DEVLINK_DPIPE_HEADER_IPV4,
++	.fields = devlink_dpipe_fields_ipv4,
++	.fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
++	.global = true,
++};
++EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv4);
++
++static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
++	{
++		.name = "destination ip",
++		.id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
++		.bitwidth = 128,
++	},
++};
++
++struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
++	.name = "ipv6",
++	.id = DEVLINK_DPIPE_HEADER_IPV6,
++	.fields = devlink_dpipe_fields_ipv6,
++	.fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
++	.global = true,
++};
++EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv6);
++
++EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
++EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
++EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
++
++static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
++	[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
++	[DEVLINK_PORT_FN_ATTR_STATE] =
++		NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
++				 DEVLINK_PORT_FN_STATE_ACTIVE),
++};
++
++static const struct nla_policy devlink_selftest_nl_policy[DEVLINK_ATTR_SELFTEST_ID_MAX + 1] = {
++	[DEVLINK_ATTR_SELFTEST_ID_FLASH] = { .type = NLA_FLAG },
++};
++
++static DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
++#define DEVLINK_REGISTERED XA_MARK_1
++#define DEVLINK_UNREGISTERING XA_MARK_2
++
++/* devlink instances are open to the access from the user space after
++ * devlink_register() call. Such logical barrier allows us to have certain
++ * expectations related to locking.
++ *
++ * Before *_register() - we are in initialization stage and no parallel
++ * access possible to the devlink instance. All drivers perform that phase
++ * by implicitly holding device_lock.
++ *
++ * After *_register() - users and driver can access devlink instance at
++ * the same time.
++ */
++#define ASSERT_DEVLINK_REGISTERED(d)                                           \
++	WARN_ON_ONCE(!xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
++#define ASSERT_DEVLINK_NOT_REGISTERED(d)                                       \
++	WARN_ON_ONCE(xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
++
++struct net *devlink_net(const struct devlink *devlink)
++{
++	return read_pnet(&devlink->_net);
++}
++EXPORT_SYMBOL_GPL(devlink_net);
++
++static void __devlink_put_rcu(struct rcu_head *head)
++{
++	struct devlink *devlink = container_of(head, struct devlink, rcu);
++
++	complete(&devlink->comp);
++}
++
++void devlink_put(struct devlink *devlink)
++{
++	if (refcount_dec_and_test(&devlink->refcount))
++		/* Make sure unregister operation that may await the completion
++		 * is unblocked only after all users are after the end of
++		 * RCU grace period.
++		 */
++		call_rcu(&devlink->rcu, __devlink_put_rcu);
++}
++
++struct devlink *__must_check devlink_try_get(struct devlink *devlink)
++{
++	if (refcount_inc_not_zero(&devlink->refcount))
++		return devlink;
++	return NULL;
++}
++
++void devl_assert_locked(struct devlink *devlink)
++{
++	lockdep_assert_held(&devlink->lock);
++}
++EXPORT_SYMBOL_GPL(devl_assert_locked);
++
++#ifdef CONFIG_LOCKDEP
++/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
++bool devl_lock_is_held(struct devlink *devlink)
++{
++	return lockdep_is_held(&devlink->lock);
++}
++EXPORT_SYMBOL_GPL(devl_lock_is_held);
++#endif
++
++void devl_lock(struct devlink *devlink)
++{
++	mutex_lock(&devlink->lock);
++}
++EXPORT_SYMBOL_GPL(devl_lock);
++
++int devl_trylock(struct devlink *devlink)
++{
++	return mutex_trylock(&devlink->lock);
++}
++EXPORT_SYMBOL_GPL(devl_trylock);
++
++void devl_unlock(struct devlink *devlink)
++{
++	mutex_unlock(&devlink->lock);
++}
++EXPORT_SYMBOL_GPL(devl_unlock);
++
++static struct devlink *
++devlinks_xa_find_get(struct net *net, unsigned long *indexp, xa_mark_t filter,
++		     void * (*xa_find_fn)(struct xarray *, unsigned long *,
++					  unsigned long, xa_mark_t))
++{
++	struct devlink *devlink;
++
++	rcu_read_lock();
++retry:
++	devlink = xa_find_fn(&devlinks, indexp, ULONG_MAX, DEVLINK_REGISTERED);
++	if (!devlink)
++		goto unlock;
++
++	/* In case devlink_unregister() was already called and "unregistering"
++	 * mark was set, do not allow to get a devlink reference here.
++	 * This prevents live-lock of devlink_unregister() wait for completion.
++	 */
++	if (xa_get_mark(&devlinks, *indexp, DEVLINK_UNREGISTERING))
++		goto retry;
++
++	/* For a possible retry, the xa_find_after() should be always used */
++	xa_find_fn = xa_find_after;
++	if (!devlink_try_get(devlink))
++		goto retry;
++	if (!net_eq(devlink_net(devlink), net)) {
++		devlink_put(devlink);
++		goto retry;
++	}
++unlock:
++	rcu_read_unlock();
++	return devlink;
++}
++
++static struct devlink *devlinks_xa_find_get_first(struct net *net,
++						  unsigned long *indexp,
++						  xa_mark_t filter)
++{
++	return devlinks_xa_find_get(net, indexp, filter, xa_find);
++}
++
++static struct devlink *devlinks_xa_find_get_next(struct net *net,
++						 unsigned long *indexp,
++						 xa_mark_t filter)
++{
++	return devlinks_xa_find_get(net, indexp, filter, xa_find_after);
++}
++
++/* Iterate over devlink pointers which were possible to get reference to.
++ * devlink_put() needs to be called for each iterated devlink pointer
++ * in loop body in order to release the reference.
++ */
++#define devlinks_xa_for_each_get(net, index, devlink, filter)			\
++	for (index = 0,								\
++	     devlink = devlinks_xa_find_get_first(net, &index, filter);		\
++	     devlink; devlink = devlinks_xa_find_get_next(net, &index, filter))
++
++#define devlinks_xa_for_each_registered_get(net, index, devlink)		\
++	devlinks_xa_for_each_get(net, index, devlink, DEVLINK_REGISTERED)
++
++static struct devlink *devlink_get_from_attrs(struct net *net,
++					      struct nlattr **attrs)
++{
++	struct devlink *devlink;
++	unsigned long index;
++	char *busname;
++	char *devname;
++
++	if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
++		return ERR_PTR(-EINVAL);
++
++	busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
++	devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
++
++	devlinks_xa_for_each_registered_get(net, index, devlink) {
++		if (strcmp(devlink->dev->bus->name, busname) == 0 &&
++		    strcmp(dev_name(devlink->dev), devname) == 0)
++			return devlink;
++		devlink_put(devlink);
++	}
++
++	return ERR_PTR(-ENODEV);
++}
++
++#define ASSERT_DEVLINK_PORT_REGISTERED(devlink_port)				\
++	WARN_ON_ONCE(!(devlink_port)->registered)
++#define ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port)			\
++	WARN_ON_ONCE((devlink_port)->registered)
++#define ASSERT_DEVLINK_PORT_INITIALIZED(devlink_port)				\
++	WARN_ON_ONCE(!(devlink_port)->initialized)
++
++static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
++						      unsigned int port_index)
++{
++	struct devlink_port *devlink_port;
++
++	list_for_each_entry(devlink_port, &devlink->port_list, list) {
++		if (devlink_port->index == port_index)
++			return devlink_port;
++	}
++	return NULL;
++}
++
++static bool devlink_port_index_exists(struct devlink *devlink,
++				      unsigned int port_index)
++{
++	return devlink_port_get_by_index(devlink, port_index);
++}
++
++static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
++							struct nlattr **attrs)
++{
++	if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
++		u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
++		struct devlink_port *devlink_port;
++
++		devlink_port = devlink_port_get_by_index(devlink, port_index);
++		if (!devlink_port)
++			return ERR_PTR(-ENODEV);
++		return devlink_port;
++	}
++	return ERR_PTR(-EINVAL);
++}
++
++static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
++						       struct genl_info *info)
++{
++	return devlink_port_get_from_attrs(devlink, info->attrs);
++}
++
++static inline bool
++devlink_rate_is_leaf(struct devlink_rate *devlink_rate)
++{
++	return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF;
++}
++
++static inline bool
++devlink_rate_is_node(struct devlink_rate *devlink_rate)
++{
++	return devlink_rate->type == DEVLINK_RATE_TYPE_NODE;
++}
++
++static struct devlink_rate *
++devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
++{
++	struct devlink_rate *devlink_rate;
++	struct devlink_port *devlink_port;
++
++	devlink_port = devlink_port_get_from_attrs(devlink, info->attrs);
++	if (IS_ERR(devlink_port))
++		return ERR_CAST(devlink_port);
++	devlink_rate = devlink_port->devlink_rate;
++	return devlink_rate ?: ERR_PTR(-ENODEV);
++}
++
++static struct devlink_rate *
++devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
++{
++	static struct devlink_rate *devlink_rate;
++
++	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
++		if (devlink_rate_is_node(devlink_rate) &&
++		    !strcmp(node_name, devlink_rate->name))
++			return devlink_rate;
++	}
++	return ERR_PTR(-ENODEV);
++}
++
++static struct devlink_rate *
++devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
++{
++	const char *rate_node_name;
++	size_t len;
++
++	if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME])
++		return ERR_PTR(-EINVAL);
++	rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]);
++	len = strlen(rate_node_name);
++	/* Name cannot be empty or decimal number */
++	if (!len || strspn(rate_node_name, "0123456789") == len)
++		return ERR_PTR(-EINVAL);
++
++	return devlink_rate_node_get_by_name(devlink, rate_node_name);
++}
++
++static struct devlink_rate *
++devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
++{
++	return devlink_rate_node_get_from_attrs(devlink, info->attrs);
++}
++
++static struct devlink_rate *
++devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
++{
++	struct nlattr **attrs = info->attrs;
++
++	if (attrs[DEVLINK_ATTR_PORT_INDEX])
++		return devlink_rate_leaf_get_from_info(devlink, info);
++	else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
++		return devlink_rate_node_get_from_info(devlink, info);
++	else
++		return ERR_PTR(-EINVAL);
++}
++
++static struct devlink_linecard *
++devlink_linecard_get_by_index(struct devlink *devlink,
++			      unsigned int linecard_index)
++{
++	struct devlink_linecard *devlink_linecard;
++
++	list_for_each_entry(devlink_linecard, &devlink->linecard_list, list) {
++		if (devlink_linecard->index == linecard_index)
++			return devlink_linecard;
++	}
++	return NULL;
++}
++
++static bool devlink_linecard_index_exists(struct devlink *devlink,
++					  unsigned int linecard_index)
++{
++	return devlink_linecard_get_by_index(devlink, linecard_index);
++}
++
++static struct devlink_linecard *
++devlink_linecard_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
++{
++	if (attrs[DEVLINK_ATTR_LINECARD_INDEX]) {
++		u32 linecard_index = nla_get_u32(attrs[DEVLINK_ATTR_LINECARD_INDEX]);
++		struct devlink_linecard *linecard;
++
++		mutex_lock(&devlink->linecards_lock);
++		linecard = devlink_linecard_get_by_index(devlink, linecard_index);
++		if (linecard)
++			refcount_inc(&linecard->refcount);
++		mutex_unlock(&devlink->linecards_lock);
++		if (!linecard)
++			return ERR_PTR(-ENODEV);
++		return linecard;
++	}
++	return ERR_PTR(-EINVAL);
++}
++
++static struct devlink_linecard *
++devlink_linecard_get_from_info(struct devlink *devlink, struct genl_info *info)
++{
++	return devlink_linecard_get_from_attrs(devlink, info->attrs);
++}
++
++static void devlink_linecard_put(struct devlink_linecard *linecard)
++{
++	if (refcount_dec_and_test(&linecard->refcount)) {
++		mutex_destroy(&linecard->state_lock);
++		kfree(linecard);
++	}
++}
++
++struct devlink_sb {
++	struct list_head list;
++	unsigned int index;
++	u32 size;
++	u16 ingress_pools_count;
++	u16 egress_pools_count;
++	u16 ingress_tc_count;
++	u16 egress_tc_count;
++};
++
++static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
++{
++	return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
++}
++
++static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
++						  unsigned int sb_index)
++{
++	struct devlink_sb *devlink_sb;
++
++	list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
++		if (devlink_sb->index == sb_index)
++			return devlink_sb;
++	}
++	return NULL;
++}
++
++static bool devlink_sb_index_exists(struct devlink *devlink,
++				    unsigned int sb_index)
++{
++	return devlink_sb_get_by_index(devlink, sb_index);
++}
++
++static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
++						    struct nlattr **attrs)
++{
++	if (attrs[DEVLINK_ATTR_SB_INDEX]) {
++		u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
++		struct devlink_sb *devlink_sb;
++
++		devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
++		if (!devlink_sb)
++			return ERR_PTR(-ENODEV);
++		return devlink_sb;
++	}
++	return ERR_PTR(-EINVAL);
++}
++
++static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
++						   struct genl_info *info)
++{
++	return devlink_sb_get_from_attrs(devlink, info->attrs);
++}
++
++static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
++						struct nlattr **attrs,
++						u16 *p_pool_index)
++{
++	u16 val;
++
++	if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
++		return -EINVAL;
++
++	val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
++	if (val >= devlink_sb_pool_count(devlink_sb))
++		return -EINVAL;
++	*p_pool_index = val;
++	return 0;
++}
++
++static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
++					       struct genl_info *info,
++					       u16 *p_pool_index)
++{
++	return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
++						    p_pool_index);
++}
++
++static int
++devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
++				    enum devlink_sb_pool_type *p_pool_type)
++{
++	u8 val;
++
++	if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
++		return -EINVAL;
++
++	val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
++	if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
++	    val != DEVLINK_SB_POOL_TYPE_EGRESS)
++		return -EINVAL;
++	*p_pool_type = val;
++	return 0;
++}
++
++static int
++devlink_sb_pool_type_get_from_info(struct genl_info *info,
++				   enum devlink_sb_pool_type *p_pool_type)
++{
++	return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
++}
++
++static int
++devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
++				  enum devlink_sb_threshold_type *p_th_type)
++{
++	u8 val;
++
++	if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
++		return -EINVAL;
++
++	val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
++	if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
++	    val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
++		return -EINVAL;
++	*p_th_type = val;
++	return 0;
++}
++
++static int
++devlink_sb_th_type_get_from_info(struct genl_info *info,
++				 enum devlink_sb_threshold_type *p_th_type)
++{
++	return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
++}
++
++static int
++devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
++				   struct nlattr **attrs,
++				   enum devlink_sb_pool_type pool_type,
++				   u16 *p_tc_index)
++{
++	u16 val;
++
++	if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
++		return -EINVAL;
++
++	val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
++	if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
++	    val >= devlink_sb->ingress_tc_count)
++		return -EINVAL;
++	if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
++	    val >= devlink_sb->egress_tc_count)
++		return -EINVAL;
++	*p_tc_index = val;
++	return 0;
++}
++
++static int
++devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
++				  struct genl_info *info,
++				  enum devlink_sb_pool_type pool_type,
++				  u16 *p_tc_index)
++{
++	return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
++						  pool_type, p_tc_index);
++}
++
++struct devlink_region {
++	struct devlink *devlink;
++	struct devlink_port *port;
++	struct list_head list;
++	union {
++		const struct devlink_region_ops *ops;
++		const struct devlink_port_region_ops *port_ops;
++	};
++	struct mutex snapshot_lock; /* protects snapshot_list,
++				     * max_snapshots and cur_snapshots
++				     * consistency.
++				     */
++	struct list_head snapshot_list;
++	u32 max_snapshots;
++	u32 cur_snapshots;
++	u64 size;
++};
++
++struct devlink_snapshot {
++	struct list_head list;
++	struct devlink_region *region;
++	u8 *data;
++	u32 id;
++};
++
++static struct devlink_region *
++devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
++{
++	struct devlink_region *region;
++
++	list_for_each_entry(region, &devlink->region_list, list)
++		if (!strcmp(region->ops->name, region_name))
++			return region;
++
++	return NULL;
++}
++
++static struct devlink_region *
++devlink_port_region_get_by_name(struct devlink_port *port,
++				const char *region_name)
++{
++	struct devlink_region *region;
++
++	list_for_each_entry(region, &port->region_list, list)
++		if (!strcmp(region->ops->name, region_name))
++			return region;
++
++	return NULL;
++}
++
++static struct devlink_snapshot *
++devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
++{
++	struct devlink_snapshot *snapshot;
++
++	list_for_each_entry(snapshot, &region->snapshot_list, list)
++		if (snapshot->id == id)
++			return snapshot;
++
++	return NULL;
++}
++
++#define DEVLINK_NL_FLAG_NEED_PORT		BIT(0)
++#define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT	BIT(1)
++#define DEVLINK_NL_FLAG_NEED_RATE		BIT(2)
++#define DEVLINK_NL_FLAG_NEED_RATE_NODE		BIT(3)
++#define DEVLINK_NL_FLAG_NEED_LINECARD		BIT(4)
++
++static int devlink_nl_pre_doit(const struct genl_ops *ops,
++			       struct sk_buff *skb, struct genl_info *info)
++{
++	struct devlink_linecard *linecard;
++	struct devlink_port *devlink_port;
++	struct devlink *devlink;
++	int err;
++
++	devlink = devlink_get_from_attrs(genl_info_net(info), info->attrs);
++	if (IS_ERR(devlink))
++		return PTR_ERR(devlink);
++	devl_lock(devlink);
++	info->user_ptr[0] = devlink;
++	if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
++		devlink_port = devlink_port_get_from_info(devlink, info);
++		if (IS_ERR(devlink_port)) {
++			err = PTR_ERR(devlink_port);
++			goto unlock;
++		}
++		info->user_ptr[1] = devlink_port;
++	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
++		devlink_port = devlink_port_get_from_info(devlink, info);
++		if (!IS_ERR(devlink_port))
++			info->user_ptr[1] = devlink_port;
++	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) {
++		struct devlink_rate *devlink_rate;
++
++		devlink_rate = devlink_rate_get_from_info(devlink, info);
++		if (IS_ERR(devlink_rate)) {
++			err = PTR_ERR(devlink_rate);
++			goto unlock;
++		}
++		info->user_ptr[1] = devlink_rate;
++	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) {
++		struct devlink_rate *rate_node;
++
++		rate_node = devlink_rate_node_get_from_info(devlink, info);
++		if (IS_ERR(rate_node)) {
++			err = PTR_ERR(rate_node);
++			goto unlock;
++		}
++		info->user_ptr[1] = rate_node;
++	} else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
++		linecard = devlink_linecard_get_from_info(devlink, info);
++		if (IS_ERR(linecard)) {
++			err = PTR_ERR(linecard);
++			goto unlock;
++		}
++		info->user_ptr[1] = linecard;
++	}
++	return 0;
++
++unlock:
++	devl_unlock(devlink);
++	devlink_put(devlink);
++	return err;
++}
++
++static void devlink_nl_post_doit(const struct genl_ops *ops,
++				 struct sk_buff *skb, struct genl_info *info)
++{
++	struct devlink_linecard *linecard;
++	struct devlink *devlink;
++
++	devlink = info->user_ptr[0];
++	if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_LINECARD) {
++		linecard = info->user_ptr[1];
++		devlink_linecard_put(linecard);
++	}
++	devl_unlock(devlink);
++	devlink_put(devlink);
++}
++
++static struct genl_family devlink_nl_family;
++
++enum devlink_multicast_groups {
++	DEVLINK_MCGRP_CONFIG,
++};
++
++static const struct genl_multicast_group devlink_nl_mcgrps[] = {
++	[DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
++};
++
++static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
++{
++	if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
++		return -EMSGSIZE;
++	if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
++		return -EMSGSIZE;
++	return 0;
++}
++
++static int devlink_nl_put_nested_handle(struct sk_buff *msg, struct devlink *devlink)
++{
++	struct nlattr *nested_attr;
++
++	nested_attr = nla_nest_start(msg, DEVLINK_ATTR_NESTED_DEVLINK);
++	if (!nested_attr)
++		return -EMSGSIZE;
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	nla_nest_end(msg, nested_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, nested_attr);
++	return -EMSGSIZE;
++}
++
++struct devlink_reload_combination {
++	enum devlink_reload_action action;
++	enum devlink_reload_limit limit;
++};
++
++static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
++	{
++		/* can't reinitialize driver with no down time */
++		.action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
++		.limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
++	},
++};
++
++static bool
++devlink_reload_combination_is_invalid(enum devlink_reload_action action,
++				      enum devlink_reload_limit limit)
++{
++	int i;
++
++	for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
++		if (devlink_reload_invalid_combinations[i].action == action &&
++		    devlink_reload_invalid_combinations[i].limit == limit)
++			return true;
++	return false;
++}
++
++static bool
++devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
++{
++	return test_bit(action, &devlink->ops->reload_actions);
++}
++
++static bool
++devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
++{
++	return test_bit(limit, &devlink->ops->reload_limits);
++}
++
++static int devlink_reload_stat_put(struct sk_buff *msg,
++				   enum devlink_reload_limit limit, u32 value)
++{
++	struct nlattr *reload_stats_entry;
++
++	reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
++	if (!reload_stats_entry)
++		return -EMSGSIZE;
++
++	if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
++	    nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
++		goto nla_put_failure;
++	nla_nest_end(msg, reload_stats_entry);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, reload_stats_entry);
++	return -EMSGSIZE;
++}
++
++static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
++{
++	struct nlattr *reload_stats_attr, *act_info, *act_stats;
++	int i, j, stat_idx;
++	u32 value;
++
++	if (!is_remote)
++		reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
++	else
++		reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
++
++	if (!reload_stats_attr)
++		return -EMSGSIZE;
++
++	for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
++		if ((!is_remote &&
++		     !devlink_reload_action_is_supported(devlink, i)) ||
++		    i == DEVLINK_RELOAD_ACTION_UNSPEC)
++			continue;
++		act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
++		if (!act_info)
++			goto nla_put_failure;
++
++		if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
++			goto action_info_nest_cancel;
++		act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
++		if (!act_stats)
++			goto action_info_nest_cancel;
++
++		for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
++			/* Remote stats are shown even if not locally supported.
++			 * Stats of actions with unspecified limit are shown
++			 * though drivers don't need to register unspecified
++			 * limit.
++			 */
++			if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
++			     !devlink_reload_limit_is_supported(devlink, j)) ||
++			    devlink_reload_combination_is_invalid(i, j))
++				continue;
++
++			stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
++			if (!is_remote)
++				value = devlink->stats.reload_stats[stat_idx];
++			else
++				value = devlink->stats.remote_reload_stats[stat_idx];
++			if (devlink_reload_stat_put(msg, j, value))
++				goto action_stats_nest_cancel;
++		}
++		nla_nest_end(msg, act_stats);
++		nla_nest_end(msg, act_info);
++	}
++	nla_nest_end(msg, reload_stats_attr);
++	return 0;
++
++action_stats_nest_cancel:
++	nla_nest_cancel(msg, act_stats);
++action_info_nest_cancel:
++	nla_nest_cancel(msg, act_info);
++nla_put_failure:
++	nla_nest_cancel(msg, reload_stats_attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
++			   enum devlink_command cmd, u32 portid,
++			   u32 seq, int flags)
++{
++	struct nlattr *dev_stats;
++	void *hdr;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
++		goto nla_put_failure;
++
++	dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
++	if (!dev_stats)
++		goto nla_put_failure;
++
++	if (devlink_reload_stats_put(msg, devlink, false))
++		goto dev_stats_nest_cancel;
++	if (devlink_reload_stats_put(msg, devlink, true))
++		goto dev_stats_nest_cancel;
++
++	nla_nest_end(msg, dev_stats);
++	genlmsg_end(msg, hdr);
++	return 0;
++
++dev_stats_nest_cancel:
++	nla_nest_cancel(msg, dev_stats);
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
++{
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
++	WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int devlink_nl_port_attrs_put(struct sk_buff *msg,
++				     struct devlink_port *devlink_port)
++{
++	struct devlink_port_attrs *attrs = &devlink_port->attrs;
++
++	if (!devlink_port->attrs_set)
++		return 0;
++	if (attrs->lanes) {
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
++			return -EMSGSIZE;
++	}
++	if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
++		return -EMSGSIZE;
++	if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
++		return -EMSGSIZE;
++	switch (devlink_port->attrs.flavour) {
++	case DEVLINK_PORT_FLAVOUR_PCI_PF:
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
++				attrs->pci_pf.controller) ||
++		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
++			return -EMSGSIZE;
++		if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
++			return -EMSGSIZE;
++		break;
++	case DEVLINK_PORT_FLAVOUR_PCI_VF:
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
++				attrs->pci_vf.controller) ||
++		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
++		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
++			return -EMSGSIZE;
++		if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
++			return -EMSGSIZE;
++		break;
++	case DEVLINK_PORT_FLAVOUR_PCI_SF:
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
++				attrs->pci_sf.controller) ||
++		    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
++				attrs->pci_sf.pf) ||
++		    nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
++				attrs->pci_sf.sf))
++			return -EMSGSIZE;
++		break;
++	case DEVLINK_PORT_FLAVOUR_PHYSICAL:
++	case DEVLINK_PORT_FLAVOUR_CPU:
++	case DEVLINK_PORT_FLAVOUR_DSA:
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
++				attrs->phys.port_number))
++			return -EMSGSIZE;
++		if (!attrs->split)
++			return 0;
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
++				attrs->phys.port_number))
++			return -EMSGSIZE;
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
++				attrs->phys.split_subport_number))
++			return -EMSGSIZE;
++		break;
++	default:
++		break;
++	}
++	return 0;
++}
++
++static int devlink_port_fn_hw_addr_fill(const struct devlink_ops *ops,
++					struct devlink_port *port,
++					struct sk_buff *msg,
++					struct netlink_ext_ack *extack,
++					bool *msg_updated)
++{
++	u8 hw_addr[MAX_ADDR_LEN];
++	int hw_addr_len;
++	int err;
++
++	if (!ops->port_function_hw_addr_get)
++		return 0;
++
++	err = ops->port_function_hw_addr_get(port, hw_addr, &hw_addr_len,
++					     extack);
++	if (err) {
++		if (err == -EOPNOTSUPP)
++			return 0;
++		return err;
++	}
++	err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
++	if (err)
++		return err;
++	*msg_updated = true;
++	return 0;
++}
++
++static int devlink_nl_rate_fill(struct sk_buff *msg,
++				struct devlink_rate *devlink_rate,
++				enum devlink_command cmd, u32 portid, u32 seq,
++				int flags, struct netlink_ext_ack *extack)
++{
++	struct devlink *devlink = devlink_rate->devlink;
++	void *hdr;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type))
++		goto nla_put_failure;
++
++	if (devlink_rate_is_leaf(devlink_rate)) {
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
++				devlink_rate->devlink_port->index))
++			goto nla_put_failure;
++	} else if (devlink_rate_is_node(devlink_rate)) {
++		if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME,
++				   devlink_rate->name))
++			goto nla_put_failure;
++	}
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE,
++			      devlink_rate->tx_share, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX,
++			      devlink_rate->tx_max, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	if (devlink_rate->parent)
++		if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
++				   devlink_rate->parent->name))
++			goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static bool
++devlink_port_fn_state_valid(enum devlink_port_fn_state state)
++{
++	return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
++	       state == DEVLINK_PORT_FN_STATE_ACTIVE;
++}
++
++static bool
++devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
++{
++	return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
++	       opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
++}
++
++static int devlink_port_fn_state_fill(const struct devlink_ops *ops,
++				      struct devlink_port *port,
++				      struct sk_buff *msg,
++				      struct netlink_ext_ack *extack,
++				      bool *msg_updated)
++{
++	enum devlink_port_fn_opstate opstate;
++	enum devlink_port_fn_state state;
++	int err;
++
++	if (!ops->port_fn_state_get)
++		return 0;
++
++	err = ops->port_fn_state_get(port, &state, &opstate, extack);
++	if (err) {
++		if (err == -EOPNOTSUPP)
++			return 0;
++		return err;
++	}
++	if (!devlink_port_fn_state_valid(state)) {
++		WARN_ON_ONCE(1);
++		NL_SET_ERR_MSG_MOD(extack, "Invalid state read from driver");
++		return -EINVAL;
++	}
++	if (!devlink_port_fn_opstate_valid(opstate)) {
++		WARN_ON_ONCE(1);
++		NL_SET_ERR_MSG_MOD(extack,
++				   "Invalid operational state read from driver");
++		return -EINVAL;
++	}
++	if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
++	    nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
++		return -EMSGSIZE;
++	*msg_updated = true;
++	return 0;
++}
++
++static int
++devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
++				   struct netlink_ext_ack *extack)
++{
++	const struct devlink_ops *ops;
++	struct nlattr *function_attr;
++	bool msg_updated = false;
++	int err;
++
++	function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
++	if (!function_attr)
++		return -EMSGSIZE;
++
++	ops = port->devlink->ops;
++	err = devlink_port_fn_hw_addr_fill(ops, port, msg, extack,
++					   &msg_updated);
++	if (err)
++		goto out;
++	err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated);
++out:
++	if (err || !msg_updated)
++		nla_nest_cancel(msg, function_attr);
++	else
++		nla_nest_end(msg, function_attr);
++	return err;
++}
++
++static int devlink_nl_port_fill(struct sk_buff *msg,
++				struct devlink_port *devlink_port,
++				enum devlink_command cmd, u32 portid, u32 seq,
++				int flags, struct netlink_ext_ack *extack)
++{
++	struct devlink *devlink = devlink_port->devlink;
++	void *hdr;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
++		goto nla_put_failure;
++
++	/* Hold rtnl lock while accessing port's netdev attributes. */
++	rtnl_lock();
++	spin_lock_bh(&devlink_port->type_lock);
++	if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
++		goto nla_put_failure_type_locked;
++	if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
++	    nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
++			devlink_port->desired_type))
++		goto nla_put_failure_type_locked;
++	if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
++		struct net *net = devlink_net(devlink_port->devlink);
++		struct net_device *netdev = devlink_port->type_dev;
++
++		if (netdev && net_eq(net, dev_net(netdev)) &&
++		    (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
++				 netdev->ifindex) ||
++		     nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
++				    netdev->name)))
++			goto nla_put_failure_type_locked;
++	}
++	if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
++		struct ib_device *ibdev = devlink_port->type_dev;
++
++		if (ibdev &&
++		    nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
++				   ibdev->name))
++			goto nla_put_failure_type_locked;
++	}
++	spin_unlock_bh(&devlink_port->type_lock);
++	rtnl_unlock();
++	if (devlink_nl_port_attrs_put(msg, devlink_port))
++		goto nla_put_failure;
++	if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
++		goto nla_put_failure;
++	if (devlink_port->linecard &&
++	    nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX,
++			devlink_port->linecard->index))
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure_type_locked:
++	spin_unlock_bh(&devlink_port->type_lock);
++	rtnl_unlock();
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static void devlink_port_notify(struct devlink_port *devlink_port,
++				enum devlink_command cmd)
++{
++	struct devlink *devlink = devlink_port->devlink;
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
++
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
++				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static void devlink_rate_notify(struct devlink_rate *devlink_rate,
++				enum devlink_command cmd)
++{
++	struct devlink *devlink = devlink_rate->devlink;
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && cmd != DEVLINK_CMD_RATE_DEL);
++
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_rate_fill(msg, devlink_rate, cmd, 0, 0, 0, NULL);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
++				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg,
++					  struct netlink_callback *cb)
++{
++	struct devlink_rate *devlink_rate;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
++			enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
++			u32 id = NETLINK_CB(cb->skb).portid;
++
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
++						   cb->nlh->nlmsg_seq,
++						   NLM_F_MULTI, NULL);
++			if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink_rate *devlink_rate = info->user_ptr[1];
++	struct sk_buff *msg;
++	int err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
++				   info->snd_portid, info->snd_seq, 0,
++				   info->extack);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static bool
++devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
++			    struct devlink_rate *parent)
++{
++	while (parent) {
++		if (parent == devlink_rate)
++			return true;
++		parent = parent->parent;
++	}
++	return false;
++}
++
++static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct sk_buff *msg;
++	int err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
++			      info->snd_portid, info->snd_seq, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
++				     struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		if (idx < start) {
++			idx++;
++			devlink_put(devlink);
++			continue;
++		}
++
++		devl_lock(devlink);
++		err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
++				      NETLINK_CB(cb->skb).portid,
++				      cb->nlh->nlmsg_seq, NLM_F_MULTI);
++		devl_unlock(devlink);
++		devlink_put(devlink);
++
++		if (err)
++			goto out;
++		idx++;
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct sk_buff *msg;
++	int err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
++				   info->snd_portid, info->snd_seq, 0,
++				   info->extack);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
++					  struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	struct devlink_port *devlink_port;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(devlink_port, &devlink->port_list, list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_port_fill(msg, devlink_port,
++						   DEVLINK_CMD_NEW,
++						   NETLINK_CB(cb->skb).portid,
++						   cb->nlh->nlmsg_seq,
++						   NLM_F_MULTI, cb->extack);
++			if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_port_type_set(struct devlink_port *devlink_port,
++				 enum devlink_port_type port_type)
++
++{
++	int err;
++
++	if (!devlink_port->devlink->ops->port_type_set)
++		return -EOPNOTSUPP;
++
++	if (port_type == devlink_port->type)
++		return 0;
++
++	err = devlink_port->devlink->ops->port_type_set(devlink_port,
++							port_type);
++	if (err)
++		return err;
++
++	devlink_port->desired_type = port_type;
++	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
++	return 0;
++}
++
++static int devlink_port_function_hw_addr_set(struct devlink_port *port,
++					     const struct nlattr *attr,
++					     struct netlink_ext_ack *extack)
++{
++	const struct devlink_ops *ops = port->devlink->ops;
++	const u8 *hw_addr;
++	int hw_addr_len;
++
++	hw_addr = nla_data(attr);
++	hw_addr_len = nla_len(attr);
++	if (hw_addr_len > MAX_ADDR_LEN) {
++		NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
++		return -EINVAL;
++	}
++	if (port->type == DEVLINK_PORT_TYPE_ETH) {
++		if (hw_addr_len != ETH_ALEN) {
++			NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
++			return -EINVAL;
++		}
++		if (!is_unicast_ether_addr(hw_addr)) {
++			NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
++			return -EINVAL;
++		}
++	}
++
++	if (!ops->port_function_hw_addr_set) {
++		NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
++		return -EOPNOTSUPP;
++	}
++
++	return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len,
++					      extack);
++}
++
++static int devlink_port_fn_state_set(struct devlink_port *port,
++				     const struct nlattr *attr,
++				     struct netlink_ext_ack *extack)
++{
++	enum devlink_port_fn_state state;
++	const struct devlink_ops *ops;
++
++	state = nla_get_u8(attr);
++	ops = port->devlink->ops;
++	if (!ops->port_fn_state_set) {
++		NL_SET_ERR_MSG_MOD(extack,
++				   "Function does not support state setting");
++		return -EOPNOTSUPP;
++	}
++	return ops->port_fn_state_set(port, state, extack);
++}
++
++static int devlink_port_function_set(struct devlink_port *port,
++				     const struct nlattr *attr,
++				     struct netlink_ext_ack *extack)
++{
++	struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
++	int err;
++
++	err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
++			       devlink_function_nl_policy, extack);
++	if (err < 0) {
++		NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
++		return err;
++	}
++
++	attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
++	if (attr) {
++		err = devlink_port_function_hw_addr_set(port, attr, extack);
++		if (err)
++			return err;
++	}
++	/* Keep this as the last function attribute set, so that when
++	 * multiple port function attributes are set along with state,
++	 * Those can be applied first before activating the state.
++	 */
++	attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
++	if (attr)
++		err = devlink_port_fn_state_set(port, attr, extack);
++
++	if (!err)
++		devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
++	return err;
++}
++
++static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	int err;
++
++	if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
++		enum devlink_port_type port_type;
++
++		port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
++		err = devlink_port_type_set(devlink_port, port_type);
++		if (err)
++			return err;
++	}
++
++	if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
++		struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
++		struct netlink_ext_ack *extack = info->extack;
++
++		err = devlink_port_function_set(devlink_port, attr, extack);
++		if (err)
++			return err;
++	}
++
++	return 0;
++}
++
++static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
++					  struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct devlink *devlink = info->user_ptr[0];
++	u32 count;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_SPLIT_COUNT))
++		return -EINVAL;
++	if (!devlink->ops->port_split)
++		return -EOPNOTSUPP;
++
++	count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
++
++	if (!devlink_port->attrs.splittable) {
++		/* Split ports cannot be split. */
++		if (devlink_port->attrs.split)
++			NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
++		else
++			NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
++		return -EINVAL;
++	}
++
++	if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
++		return -EINVAL;
++	}
++
++	return devlink->ops->port_split(devlink, devlink_port, count,
++					info->extack);
++}
++
++static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
++					    struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct devlink *devlink = info->user_ptr[0];
++
++	if (!devlink->ops->port_unsplit)
++		return -EOPNOTSUPP;
++	return devlink->ops->port_unsplit(devlink, devlink_port, info->extack);
++}
++
++static int devlink_port_new_notify(struct devlink *devlink,
++				   unsigned int port_index,
++				   struct genl_info *info)
++{
++	struct devlink_port *devlink_port;
++	struct sk_buff *msg;
++	int err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	lockdep_assert_held(&devlink->lock);
++	devlink_port = devlink_port_get_by_index(devlink, port_index);
++	if (!devlink_port) {
++		err = -ENODEV;
++		goto out;
++	}
++
++	err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_NEW,
++				   info->snd_portid, info->snd_seq, 0, NULL);
++	if (err)
++		goto out;
++
++	return genlmsg_reply(msg, info);
++
++out:
++	nlmsg_free(msg);
++	return err;
++}
++
++static int devlink_nl_cmd_port_new_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink_port_new_attrs new_attrs = {};
++	struct devlink *devlink = info->user_ptr[0];
++	unsigned int new_port_index;
++	int err;
++
++	if (!devlink->ops->port_new || !devlink->ops->port_del)
++		return -EOPNOTSUPP;
++
++	if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
++	    !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
++		NL_SET_ERR_MSG_MOD(extack, "Port flavour or PCI PF are not specified");
++		return -EINVAL;
++	}
++	new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
++	new_attrs.pfnum =
++		nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);
++
++	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
++		/* Port index of the new port being created by driver. */
++		new_attrs.port_index =
++			nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
++		new_attrs.port_index_valid = true;
++	}
++	if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
++		new_attrs.controller =
++			nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
++		new_attrs.controller_valid = true;
++	}
++	if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
++	    info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
++		new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
++		new_attrs.sfnum_valid = true;
++	}
++
++	err = devlink->ops->port_new(devlink, &new_attrs, extack,
++				     &new_port_index);
++	if (err)
++		return err;
++
++	err = devlink_port_new_notify(devlink, new_port_index, info);
++	if (err && err != -ENODEV) {
++		/* Fail to send the response; destroy newly created port. */
++		devlink->ops->port_del(devlink, new_port_index, extack);
++	}
++	return err;
++}
++
++static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++	unsigned int port_index;
++
++	if (!devlink->ops->port_del)
++		return -EOPNOTSUPP;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_INDEX)) {
++		NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
++		return -EINVAL;
++	}
++	port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
++
++	return devlink->ops->port_del(devlink, port_index, extack);
++}
++
++static int
++devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
++				struct genl_info *info,
++				struct nlattr *nla_parent)
++{
++	struct devlink *devlink = devlink_rate->devlink;
++	const char *parent_name = nla_data(nla_parent);
++	const struct devlink_ops *ops = devlink->ops;
++	size_t len = strlen(parent_name);
++	struct devlink_rate *parent;
++	int err = -EOPNOTSUPP;
++
++	parent = devlink_rate->parent;
++	if (parent && len) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent.");
++		return -EBUSY;
++	} else if (parent && !len) {
++		if (devlink_rate_is_leaf(devlink_rate))
++			err = ops->rate_leaf_parent_set(devlink_rate, NULL,
++							devlink_rate->priv, NULL,
++							info->extack);
++		else if (devlink_rate_is_node(devlink_rate))
++			err = ops->rate_node_parent_set(devlink_rate, NULL,
++							devlink_rate->priv, NULL,
++							info->extack);
++		if (err)
++			return err;
++
++		refcount_dec(&parent->refcnt);
++		devlink_rate->parent = NULL;
++	} else if (!parent && len) {
++		parent = devlink_rate_node_get_by_name(devlink, parent_name);
++		if (IS_ERR(parent))
++			return -ENODEV;
++
++		if (parent == devlink_rate) {
++			NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed");
++			return -EINVAL;
++		}
++
++		if (devlink_rate_is_node(devlink_rate) &&
++		    devlink_rate_is_parent_node(devlink_rate, parent->parent)) {
++			NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node.");
++			return -EEXIST;
++		}
++
++		if (devlink_rate_is_leaf(devlink_rate))
++			err = ops->rate_leaf_parent_set(devlink_rate, parent,
++							devlink_rate->priv, parent->priv,
++							info->extack);
++		else if (devlink_rate_is_node(devlink_rate))
++			err = ops->rate_node_parent_set(devlink_rate, parent,
++							devlink_rate->priv, parent->priv,
++							info->extack);
++		if (err)
++			return err;
++
++		refcount_inc(&parent->refcnt);
++		devlink_rate->parent = parent;
++	}
++
++	return 0;
++}
++
++static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
++			       const struct devlink_ops *ops,
++			       struct genl_info *info)
++{
++	struct nlattr *nla_parent, **attrs = info->attrs;
++	int err = -EOPNOTSUPP;
++	u64 rate;
++
++	if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) {
++		rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]);
++		if (devlink_rate_is_leaf(devlink_rate))
++			err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv,
++							  rate, info->extack);
++		else if (devlink_rate_is_node(devlink_rate))
++			err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv,
++							  rate, info->extack);
++		if (err)
++			return err;
++		devlink_rate->tx_share = rate;
++	}
++
++	if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) {
++		rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]);
++		if (devlink_rate_is_leaf(devlink_rate))
++			err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv,
++							rate, info->extack);
++		else if (devlink_rate_is_node(devlink_rate))
++			err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv,
++							rate, info->extack);
++		if (err)
++			return err;
++		devlink_rate->tx_max = rate;
++	}
++
++	nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
++	if (nla_parent) {
++		err = devlink_nl_rate_parent_node_set(devlink_rate, info,
++						      nla_parent);
++		if (err)
++			return err;
++	}
++
++	return 0;
++}
++
++static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
++					   struct genl_info *info,
++					   enum devlink_rate_type type)
++{
++	struct nlattr **attrs = info->attrs;
++
++	if (type == DEVLINK_RATE_TYPE_LEAF) {
++		if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) {
++			NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs");
++			return false;
++		}
++		if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) {
++			NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs");
++			return false;
++		}
++		if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
++		    !ops->rate_leaf_parent_set) {
++			NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs");
++			return false;
++		}
++	} else if (type == DEVLINK_RATE_TYPE_NODE) {
++		if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) {
++			NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes");
++			return false;
++		}
++		if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) {
++			NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes");
++			return false;
++		}
++		if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
++		    !ops->rate_node_parent_set) {
++			NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes");
++			return false;
++		}
++	} else {
++		WARN(1, "Unknown type of rate object");
++		return false;
++	}
++
++	return true;
++}
++
++static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink_rate *devlink_rate = info->user_ptr[1];
++	struct devlink *devlink = devlink_rate->devlink;
++	const struct devlink_ops *ops = devlink->ops;
++	int err;
++
++	if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
++		return -EOPNOTSUPP;
++
++	err = devlink_nl_rate_set(devlink_rate, ops, info);
++
++	if (!err)
++		devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
++	return err;
++}
++
++static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_rate *rate_node;
++	const struct devlink_ops *ops;
++	int err;
++
++	ops = devlink->ops;
++	if (!ops || !ops->rate_node_new || !ops->rate_node_del) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported");
++		return -EOPNOTSUPP;
++	}
++
++	if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
++		return -EOPNOTSUPP;
++
++	rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
++	if (!IS_ERR(rate_node))
++		return -EEXIST;
++	else if (rate_node == ERR_PTR(-EINVAL))
++		return -EINVAL;
++
++	rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL);
++	if (!rate_node)
++		return -ENOMEM;
++
++	rate_node->devlink = devlink;
++	rate_node->type = DEVLINK_RATE_TYPE_NODE;
++	rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL);
++	if (!rate_node->name) {
++		err = -ENOMEM;
++		goto err_strdup;
++	}
++
++	err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack);
++	if (err)
++		goto err_node_new;
++
++	err = devlink_nl_rate_set(rate_node, ops, info);
++	if (err)
++		goto err_rate_set;
++
++	refcount_set(&rate_node->refcnt, 1);
++	list_add(&rate_node->list, &devlink->rate_list);
++	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
++	return 0;
++
++err_rate_set:
++	ops->rate_node_del(rate_node, rate_node->priv, info->extack);
++err_node_new:
++	kfree(rate_node->name);
++err_strdup:
++	kfree(rate_node);
++	return err;
++}
++
++static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink_rate *rate_node = info->user_ptr[1];
++	struct devlink *devlink = rate_node->devlink;
++	const struct devlink_ops *ops = devlink->ops;
++	int err;
++
++	if (refcount_read(&rate_node->refcnt) > 1) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node.");
++		return -EBUSY;
++	}
++
++	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
++	err = ops->rate_node_del(rate_node, rate_node->priv, info->extack);
++	if (rate_node->parent)
++		refcount_dec(&rate_node->parent->refcnt);
++	list_del(&rate_node->list);
++	kfree(rate_node->name);
++	kfree(rate_node);
++	return err;
++}
++
++struct devlink_linecard_type {
++	const char *type;
++	const void *priv;
++};
++
++static int devlink_nl_linecard_fill(struct sk_buff *msg,
++				    struct devlink *devlink,
++				    struct devlink_linecard *linecard,
++				    enum devlink_command cmd, u32 portid,
++				    u32 seq, int flags,
++				    struct netlink_ext_ack *extack)
++{
++	struct devlink_linecard_type *linecard_type;
++	struct nlattr *attr;
++	void *hdr;
++	int i;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, linecard->index))
++		goto nla_put_failure;
++	if (nla_put_u8(msg, DEVLINK_ATTR_LINECARD_STATE, linecard->state))
++		goto nla_put_failure;
++	if (linecard->type &&
++	    nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE, linecard->type))
++		goto nla_put_failure;
++
++	if (linecard->types_count) {
++		attr = nla_nest_start(msg,
++				      DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES);
++		if (!attr)
++			goto nla_put_failure;
++		for (i = 0; i < linecard->types_count; i++) {
++			linecard_type = &linecard->types[i];
++			if (nla_put_string(msg, DEVLINK_ATTR_LINECARD_TYPE,
++					   linecard_type->type)) {
++				nla_nest_cancel(msg, attr);
++				goto nla_put_failure;
++			}
++		}
++		nla_nest_end(msg, attr);
++	}
++
++	if (linecard->nested_devlink &&
++	    devlink_nl_put_nested_handle(msg, linecard->nested_devlink))
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static void devlink_linecard_notify(struct devlink_linecard *linecard,
++				    enum devlink_command cmd)
++{
++	struct devlink *devlink = linecard->devlink;
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_LINECARD_NEW &&
++		cmd != DEVLINK_CMD_LINECARD_DEL);
++
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_linecard_fill(msg, devlink, linecard, cmd, 0, 0, 0,
++				       NULL);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int devlink_nl_cmd_linecard_get_doit(struct sk_buff *skb,
++					    struct genl_info *info)
++{
++	struct devlink_linecard *linecard = info->user_ptr[1];
++	struct devlink *devlink = linecard->devlink;
++	struct sk_buff *msg;
++	int err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	mutex_lock(&linecard->state_lock);
++	err = devlink_nl_linecard_fill(msg, devlink, linecard,
++				       DEVLINK_CMD_LINECARD_NEW,
++				       info->snd_portid, info->snd_seq, 0,
++				       info->extack);
++	mutex_unlock(&linecard->state_lock);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_linecard_get_dumpit(struct sk_buff *msg,
++					      struct netlink_callback *cb)
++{
++	struct devlink_linecard *linecard;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		mutex_lock(&devlink->linecards_lock);
++		list_for_each_entry(linecard, &devlink->linecard_list, list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			mutex_lock(&linecard->state_lock);
++			err = devlink_nl_linecard_fill(msg, devlink, linecard,
++						       DEVLINK_CMD_LINECARD_NEW,
++						       NETLINK_CB(cb->skb).portid,
++						       cb->nlh->nlmsg_seq,
++						       NLM_F_MULTI,
++						       cb->extack);
++			mutex_unlock(&linecard->state_lock);
++			if (err) {
++				mutex_unlock(&devlink->linecards_lock);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		mutex_unlock(&devlink->linecards_lock);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static struct devlink_linecard_type *
++devlink_linecard_type_lookup(struct devlink_linecard *linecard,
++			     const char *type)
++{
++	struct devlink_linecard_type *linecard_type;
++	int i;
++
++	for (i = 0; i < linecard->types_count; i++) {
++		linecard_type = &linecard->types[i];
++		if (!strcmp(type, linecard_type->type))
++			return linecard_type;
++	}
++	return NULL;
++}
++
++static int devlink_linecard_type_set(struct devlink_linecard *linecard,
++				     const char *type,
++				     struct netlink_ext_ack *extack)
++{
++	const struct devlink_linecard_ops *ops = linecard->ops;
++	struct devlink_linecard_type *linecard_type;
++	int err;
++
++	mutex_lock(&linecard->state_lock);
++	if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
++		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
++		err = -EBUSY;
++		goto out;
++	}
++	if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
++		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
++		err = -EBUSY;
++		goto out;
++	}
++
++	linecard_type = devlink_linecard_type_lookup(linecard, type);
++	if (!linecard_type) {
++		NL_SET_ERR_MSG_MOD(extack, "Unsupported line card type provided");
++		err = -EINVAL;
++		goto out;
++	}
++
++	if (linecard->state != DEVLINK_LINECARD_STATE_UNPROVISIONED &&
++	    linecard->state != DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
++		NL_SET_ERR_MSG_MOD(extack, "Line card already provisioned");
++		err = -EBUSY;
++		/* Check if the line card is provisioned in the same
++		 * way the user asks. In case it is, make the operation
++		 * to return success.
++		 */
++		if (ops->same_provision &&
++		    ops->same_provision(linecard, linecard->priv,
++					linecard_type->type,
++					linecard_type->priv))
++			err = 0;
++		goto out;
++	}
++
++	linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING;
++	linecard->type = linecard_type->type;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++	err = ops->provision(linecard, linecard->priv, linecard_type->type,
++			     linecard_type->priv, extack);
++	if (err) {
++		/* Provisioning failed. Assume the linecard is unprovisioned
++		 * for future operations.
++		 */
++		mutex_lock(&linecard->state_lock);
++		linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
++		linecard->type = NULL;
++		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++		mutex_unlock(&linecard->state_lock);
++	}
++	return err;
++
++out:
++	mutex_unlock(&linecard->state_lock);
++	return err;
++}
++
++static int devlink_linecard_type_unset(struct devlink_linecard *linecard,
++				       struct netlink_ext_ack *extack)
++{
++	int err;
++
++	mutex_lock(&linecard->state_lock);
++	if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING) {
++		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being provisioned");
++		err = -EBUSY;
++		goto out;
++	}
++	if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONING) {
++		NL_SET_ERR_MSG_MOD(extack, "Line card is currently being unprovisioned");
++		err = -EBUSY;
++		goto out;
++	}
++	if (linecard->state == DEVLINK_LINECARD_STATE_PROVISIONING_FAILED) {
++		linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
++		linecard->type = NULL;
++		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++		err = 0;
++		goto out;
++	}
++
++	if (linecard->state == DEVLINK_LINECARD_STATE_UNPROVISIONED) {
++		NL_SET_ERR_MSG_MOD(extack, "Line card is not provisioned");
++		err = 0;
++		goto out;
++	}
++	linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONING;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++	err = linecard->ops->unprovision(linecard, linecard->priv,
++					 extack);
++	if (err) {
++		/* Unprovisioning failed. Assume the linecard is unprovisioned
++		 * for future operations.
++		 */
++		mutex_lock(&linecard->state_lock);
++		linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
++		linecard->type = NULL;
++		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++		mutex_unlock(&linecard->state_lock);
++	}
++	return err;
++
++out:
++	mutex_unlock(&linecard->state_lock);
++	return err;
++}
++
++static int devlink_nl_cmd_linecard_set_doit(struct sk_buff *skb,
++					    struct genl_info *info)
++{
++	struct devlink_linecard *linecard = info->user_ptr[1];
++	struct netlink_ext_ack *extack = info->extack;
++	int err;
++
++	if (info->attrs[DEVLINK_ATTR_LINECARD_TYPE]) {
++		const char *type;
++
++		type = nla_data(info->attrs[DEVLINK_ATTR_LINECARD_TYPE]);
++		if (strcmp(type, "")) {
++			err = devlink_linecard_type_set(linecard, type, extack);
++			if (err)
++				return err;
++		} else {
++			err = devlink_linecard_type_unset(linecard, extack);
++			if (err)
++				return err;
++		}
++	}
++
++	return 0;
++}
++
++static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
++			      struct devlink_sb *devlink_sb,
++			      enum devlink_command cmd, u32 portid,
++			      u32 seq, int flags)
++{
++	void *hdr;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
++			devlink_sb->ingress_pools_count))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
++			devlink_sb->egress_pools_count))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
++			devlink_sb->ingress_tc_count))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
++			devlink_sb->egress_tc_count))
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
++				      struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_sb *devlink_sb;
++	struct sk_buff *msg;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
++				 DEVLINK_CMD_SB_NEW,
++				 info->snd_portid, info->snd_seq, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
++					struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	struct devlink_sb *devlink_sb;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
++						 DEVLINK_CMD_SB_NEW,
++						 NETLINK_CB(cb->skb).portid,
++						 cb->nlh->nlmsg_seq,
++						 NLM_F_MULTI);
++			if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
++				   struct devlink_sb *devlink_sb,
++				   u16 pool_index, enum devlink_command cmd,
++				   u32 portid, u32 seq, int flags)
++{
++	struct devlink_sb_pool_info pool_info;
++	void *hdr;
++	int err;
++
++	err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
++					pool_index, &pool_info);
++	if (err)
++		return err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
++		goto nla_put_failure;
++	if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
++		goto nla_put_failure;
++	if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
++		       pool_info.threshold_type))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
++			pool_info.cell_size))
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
++					   struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_sb *devlink_sb;
++	struct sk_buff *msg;
++	u16 pool_index;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
++						  &pool_index);
++	if (err)
++		return err;
++
++	if (!devlink->ops->sb_pool_get)
++		return -EOPNOTSUPP;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
++				      DEVLINK_CMD_SB_POOL_NEW,
++				      info->snd_portid, info->snd_seq, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
++				struct devlink *devlink,
++				struct devlink_sb *devlink_sb,
++				u32 portid, u32 seq)
++{
++	u16 pool_count = devlink_sb_pool_count(devlink_sb);
++	u16 pool_index;
++	int err;
++
++	for (pool_index = 0; pool_index < pool_count; pool_index++) {
++		if (*p_idx < start) {
++			(*p_idx)++;
++			continue;
++		}
++		err = devlink_nl_sb_pool_fill(msg, devlink,
++					      devlink_sb,
++					      pool_index,
++					      DEVLINK_CMD_SB_POOL_NEW,
++					      portid, seq, NLM_F_MULTI);
++		if (err)
++			return err;
++		(*p_idx)++;
++	}
++	return 0;
++}
++
++static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
++					     struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	struct devlink_sb *devlink_sb;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		if (!devlink->ops->sb_pool_get)
++			goto retry;
++
++		devl_lock(devlink);
++		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
++			err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
++						   devlink_sb,
++						   NETLINK_CB(cb->skb).portid,
++						   cb->nlh->nlmsg_seq);
++			if (err == -EOPNOTSUPP) {
++				err = 0;
++			} else if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++		}
++		devl_unlock(devlink);
++retry:
++		devlink_put(devlink);
++	}
++out:
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
++			       u16 pool_index, u32 size,
++			       enum devlink_sb_threshold_type threshold_type,
++			       struct netlink_ext_ack *extack)
++
++{
++	const struct devlink_ops *ops = devlink->ops;
++
++	if (ops->sb_pool_set)
++		return ops->sb_pool_set(devlink, sb_index, pool_index,
++					size, threshold_type, extack);
++	return -EOPNOTSUPP;
++}
++
++static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
++					   struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	enum devlink_sb_threshold_type threshold_type;
++	struct devlink_sb *devlink_sb;
++	u16 pool_index;
++	u32 size;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
++						  &pool_index);
++	if (err)
++		return err;
++
++	err = devlink_sb_th_type_get_from_info(info, &threshold_type);
++	if (err)
++		return err;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_POOL_SIZE))
++		return -EINVAL;
++
++	size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
++	return devlink_sb_pool_set(devlink, devlink_sb->index,
++				   pool_index, size, threshold_type,
++				   info->extack);
++}
++
++static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
++					struct devlink *devlink,
++					struct devlink_port *devlink_port,
++					struct devlink_sb *devlink_sb,
++					u16 pool_index,
++					enum devlink_command cmd,
++					u32 portid, u32 seq, int flags)
++{
++	const struct devlink_ops *ops = devlink->ops;
++	u32 threshold;
++	void *hdr;
++	int err;
++
++	err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
++				    pool_index, &threshold);
++	if (err)
++		return err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
++		goto nla_put_failure;
++
++	if (ops->sb_occ_port_pool_get) {
++		u32 cur;
++		u32 max;
++
++		err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
++						pool_index, &cur, &max);
++		if (err && err != -EOPNOTSUPP)
++			goto sb_occ_get_failure;
++		if (!err) {
++			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
++				goto nla_put_failure;
++			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
++				goto nla_put_failure;
++		}
++	}
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	err = -EMSGSIZE;
++sb_occ_get_failure:
++	genlmsg_cancel(msg, hdr);
++	return err;
++}
++
++static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
++						struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct devlink *devlink = devlink_port->devlink;
++	struct devlink_sb *devlink_sb;
++	struct sk_buff *msg;
++	u16 pool_index;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
++						  &pool_index);
++	if (err)
++		return err;
++
++	if (!devlink->ops->sb_port_pool_get)
++		return -EOPNOTSUPP;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
++					   devlink_sb, pool_index,
++					   DEVLINK_CMD_SB_PORT_POOL_NEW,
++					   info->snd_portid, info->snd_seq, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
++				     struct devlink *devlink,
++				     struct devlink_sb *devlink_sb,
++				     u32 portid, u32 seq)
++{
++	struct devlink_port *devlink_port;
++	u16 pool_count = devlink_sb_pool_count(devlink_sb);
++	u16 pool_index;
++	int err;
++
++	list_for_each_entry(devlink_port, &devlink->port_list, list) {
++		for (pool_index = 0; pool_index < pool_count; pool_index++) {
++			if (*p_idx < start) {
++				(*p_idx)++;
++				continue;
++			}
++			err = devlink_nl_sb_port_pool_fill(msg, devlink,
++							   devlink_port,
++							   devlink_sb,
++							   pool_index,
++							   DEVLINK_CMD_SB_PORT_POOL_NEW,
++							   portid, seq,
++							   NLM_F_MULTI);
++			if (err)
++				return err;
++			(*p_idx)++;
++		}
++	}
++	return 0;
++}
++
++static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
++						  struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	struct devlink_sb *devlink_sb;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		if (!devlink->ops->sb_port_pool_get)
++			goto retry;
++
++		devl_lock(devlink);
++		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
++			err = __sb_port_pool_get_dumpit(msg, start, &idx,
++							devlink, devlink_sb,
++							NETLINK_CB(cb->skb).portid,
++							cb->nlh->nlmsg_seq);
++			if (err == -EOPNOTSUPP) {
++				err = 0;
++			} else if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++		}
++		devl_unlock(devlink);
++retry:
++		devlink_put(devlink);
++	}
++out:
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
++				    unsigned int sb_index, u16 pool_index,
++				    u32 threshold,
++				    struct netlink_ext_ack *extack)
++
++{
++	const struct devlink_ops *ops = devlink_port->devlink->ops;
++
++	if (ops->sb_port_pool_set)
++		return ops->sb_port_pool_set(devlink_port, sb_index,
++					     pool_index, threshold, extack);
++	return -EOPNOTSUPP;
++}
++
++static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
++						struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_sb *devlink_sb;
++	u16 pool_index;
++	u32 threshold;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
++						  &pool_index);
++	if (err)
++		return err;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD))
++		return -EINVAL;
++
++	threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
++	return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
++					pool_index, threshold, info->extack);
++}
++
++static int
++devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
++				struct devlink_port *devlink_port,
++				struct devlink_sb *devlink_sb, u16 tc_index,
++				enum devlink_sb_pool_type pool_type,
++				enum devlink_command cmd,
++				u32 portid, u32 seq, int flags)
++{
++	const struct devlink_ops *ops = devlink->ops;
++	u16 pool_index;
++	u32 threshold;
++	void *hdr;
++	int err;
++
++	err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
++				       tc_index, pool_type,
++				       &pool_index, &threshold);
++	if (err)
++		return err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
++		goto nla_put_failure;
++	if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
++		goto nla_put_failure;
++	if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
++		goto nla_put_failure;
++	if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
++		goto nla_put_failure;
++
++	if (ops->sb_occ_tc_port_bind_get) {
++		u32 cur;
++		u32 max;
++
++		err = ops->sb_occ_tc_port_bind_get(devlink_port,
++						   devlink_sb->index,
++						   tc_index, pool_type,
++						   &cur, &max);
++		if (err && err != -EOPNOTSUPP)
++			return err;
++		if (!err) {
++			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
++				goto nla_put_failure;
++			if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
++				goto nla_put_failure;
++		}
++	}
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
++						   struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct devlink *devlink = devlink_port->devlink;
++	struct devlink_sb *devlink_sb;
++	struct sk_buff *msg;
++	enum devlink_sb_pool_type pool_type;
++	u16 tc_index;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	err = devlink_sb_pool_type_get_from_info(info, &pool_type);
++	if (err)
++		return err;
++
++	err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
++						pool_type, &tc_index);
++	if (err)
++		return err;
++
++	if (!devlink->ops->sb_tc_pool_bind_get)
++		return -EOPNOTSUPP;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
++					      devlink_sb, tc_index, pool_type,
++					      DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
++					      info->snd_portid,
++					      info->snd_seq, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
++					int start, int *p_idx,
++					struct devlink *devlink,
++					struct devlink_sb *devlink_sb,
++					u32 portid, u32 seq)
++{
++	struct devlink_port *devlink_port;
++	u16 tc_index;
++	int err;
++
++	list_for_each_entry(devlink_port, &devlink->port_list, list) {
++		for (tc_index = 0;
++		     tc_index < devlink_sb->ingress_tc_count; tc_index++) {
++			if (*p_idx < start) {
++				(*p_idx)++;
++				continue;
++			}
++			err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
++							      devlink_port,
++							      devlink_sb,
++							      tc_index,
++							      DEVLINK_SB_POOL_TYPE_INGRESS,
++							      DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
++							      portid, seq,
++							      NLM_F_MULTI);
++			if (err)
++				return err;
++			(*p_idx)++;
++		}
++		for (tc_index = 0;
++		     tc_index < devlink_sb->egress_tc_count; tc_index++) {
++			if (*p_idx < start) {
++				(*p_idx)++;
++				continue;
++			}
++			err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
++							      devlink_port,
++							      devlink_sb,
++							      tc_index,
++							      DEVLINK_SB_POOL_TYPE_EGRESS,
++							      DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
++							      portid, seq,
++							      NLM_F_MULTI);
++			if (err)
++				return err;
++			(*p_idx)++;
++		}
++	}
++	return 0;
++}
++
++static int
++devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
++					  struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	struct devlink_sb *devlink_sb;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		if (!devlink->ops->sb_tc_pool_bind_get)
++			goto retry;
++
++		devl_lock(devlink);
++		list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
++			err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
++							   devlink,
++							   devlink_sb,
++							   NETLINK_CB(cb->skb).portid,
++							   cb->nlh->nlmsg_seq);
++			if (err == -EOPNOTSUPP) {
++				err = 0;
++			} else if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++		}
++		devl_unlock(devlink);
++retry:
++		devlink_put(devlink);
++	}
++out:
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
++				       unsigned int sb_index, u16 tc_index,
++				       enum devlink_sb_pool_type pool_type,
++				       u16 pool_index, u32 threshold,
++				       struct netlink_ext_ack *extack)
++
++{
++	const struct devlink_ops *ops = devlink_port->devlink->ops;
++
++	if (ops->sb_tc_pool_bind_set)
++		return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
++						tc_index, pool_type,
++						pool_index, threshold, extack);
++	return -EOPNOTSUPP;
++}
++
++static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
++						   struct genl_info *info)
++{
++	struct devlink_port *devlink_port = info->user_ptr[1];
++	struct devlink *devlink = info->user_ptr[0];
++	enum devlink_sb_pool_type pool_type;
++	struct devlink_sb *devlink_sb;
++	u16 tc_index;
++	u16 pool_index;
++	u32 threshold;
++	int err;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	err = devlink_sb_pool_type_get_from_info(info, &pool_type);
++	if (err)
++		return err;
++
++	err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
++						pool_type, &tc_index);
++	if (err)
++		return err;
++
++	err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
++						  &pool_index);
++	if (err)
++		return err;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD))
++		return -EINVAL;
++
++	threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
++	return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
++					   tc_index, pool_type,
++					   pool_index, threshold, info->extack);
++}
++
++static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
++					       struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	const struct devlink_ops *ops = devlink->ops;
++	struct devlink_sb *devlink_sb;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	if (ops->sb_occ_snapshot)
++		return ops->sb_occ_snapshot(devlink, devlink_sb->index);
++	return -EOPNOTSUPP;
++}
++
++static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
++						struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	const struct devlink_ops *ops = devlink->ops;
++	struct devlink_sb *devlink_sb;
++
++	devlink_sb = devlink_sb_get_from_info(devlink, info);
++	if (IS_ERR(devlink_sb))
++		return PTR_ERR(devlink_sb);
++
++	if (ops->sb_occ_max_clear)
++		return ops->sb_occ_max_clear(devlink, devlink_sb->index);
++	return -EOPNOTSUPP;
++}
++
++static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
++				   enum devlink_command cmd, u32 portid,
++				   u32 seq, int flags)
++{
++	const struct devlink_ops *ops = devlink->ops;
++	enum devlink_eswitch_encap_mode encap_mode;
++	u8 inline_mode;
++	void *hdr;
++	int err = 0;
++	u16 mode;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	err = devlink_nl_put_handle(msg, devlink);
++	if (err)
++		goto nla_put_failure;
++
++	if (ops->eswitch_mode_get) {
++		err = ops->eswitch_mode_get(devlink, &mode);
++		if (err)
++			goto nla_put_failure;
++		err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
++		if (err)
++			goto nla_put_failure;
++	}
++
++	if (ops->eswitch_inline_mode_get) {
++		err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
++		if (err)
++			goto nla_put_failure;
++		err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
++				 inline_mode);
++		if (err)
++			goto nla_put_failure;
++	}
++
++	if (ops->eswitch_encap_mode_get) {
++		err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
++		if (err)
++			goto nla_put_failure;
++		err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
++		if (err)
++			goto nla_put_failure;
++	}
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return err;
++}
++
++static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
++					   struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct sk_buff *msg;
++	int err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
++				      info->snd_portid, info->snd_seq, 0);
++
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode,
++				    struct netlink_ext_ack *extack)
++{
++	struct devlink_rate *devlink_rate;
++
++	list_for_each_entry(devlink_rate, &devlink->rate_list, list)
++		if (devlink_rate_is_node(devlink_rate)) {
++			NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists.");
++			return -EBUSY;
++		}
++	return 0;
++}
++
++static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
++					   struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	const struct devlink_ops *ops = devlink->ops;
++	enum devlink_eswitch_encap_mode encap_mode;
++	u8 inline_mode;
++	int err = 0;
++	u16 mode;
++
++	if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
++		if (!ops->eswitch_mode_set)
++			return -EOPNOTSUPP;
++		mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
++		err = devlink_rate_nodes_check(devlink, mode, info->extack);
++		if (err)
++			return err;
++		err = ops->eswitch_mode_set(devlink, mode, info->extack);
++		if (err)
++			return err;
++	}
++
++	if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
++		if (!ops->eswitch_inline_mode_set)
++			return -EOPNOTSUPP;
++		inline_mode = nla_get_u8(
++				info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
++		err = ops->eswitch_inline_mode_set(devlink, inline_mode,
++						   info->extack);
++		if (err)
++			return err;
++	}
++
++	if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
++		if (!ops->eswitch_encap_mode_set)
++			return -EOPNOTSUPP;
++		encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
++		err = ops->eswitch_encap_mode_set(devlink, encap_mode,
++						  info->extack);
++		if (err)
++			return err;
++	}
++
++	return 0;
++}
++
++int devlink_dpipe_match_put(struct sk_buff *skb,
++			    struct devlink_dpipe_match *match)
++{
++	struct devlink_dpipe_header *header = match->header;
++	struct devlink_dpipe_field *field = &header->fields[match->field_id];
++	struct nlattr *match_attr;
++
++	match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
++	if (!match_attr)
++		return -EMSGSIZE;
++
++	if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
++	    nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
++		goto nla_put_failure;
++
++	nla_nest_end(skb, match_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, match_attr);
++	return -EMSGSIZE;
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
++
++static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
++				     struct sk_buff *skb)
++{
++	struct nlattr *matches_attr;
++
++	matches_attr = nla_nest_start_noflag(skb,
++					     DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
++	if (!matches_attr)
++		return -EMSGSIZE;
++
++	if (table->table_ops->matches_dump(table->priv, skb))
++		goto nla_put_failure;
++
++	nla_nest_end(skb, matches_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, matches_attr);
++	return -EMSGSIZE;
++}
++
++int devlink_dpipe_action_put(struct sk_buff *skb,
++			     struct devlink_dpipe_action *action)
++{
++	struct devlink_dpipe_header *header = action->header;
++	struct devlink_dpipe_field *field = &header->fields[action->field_id];
++	struct nlattr *action_attr;
++
++	action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
++	if (!action_attr)
++		return -EMSGSIZE;
++
++	if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
++	    nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
++		goto nla_put_failure;
++
++	nla_nest_end(skb, action_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, action_attr);
++	return -EMSGSIZE;
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
++
++static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
++				     struct sk_buff *skb)
++{
++	struct nlattr *actions_attr;
++
++	actions_attr = nla_nest_start_noflag(skb,
++					     DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
++	if (!actions_attr)
++		return -EMSGSIZE;
++
++	if (table->table_ops->actions_dump(table->priv, skb))
++		goto nla_put_failure;
++
++	nla_nest_end(skb, actions_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, actions_attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_dpipe_table_put(struct sk_buff *skb,
++				   struct devlink_dpipe_table *table)
++{
++	struct nlattr *table_attr;
++	u64 table_size;
++
++	table_size = table->table_ops->size_get(table->priv);
++	table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
++	if (!table_attr)
++		return -EMSGSIZE;
++
++	if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
++	    nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++	if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
++		       table->counters_enabled))
++		goto nla_put_failure;
++
++	if (table->resource_valid) {
++		if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
++				      table->resource_id, DEVLINK_ATTR_PAD) ||
++		    nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
++				      table->resource_units, DEVLINK_ATTR_PAD))
++			goto nla_put_failure;
++	}
++	if (devlink_dpipe_matches_put(table, skb))
++		goto nla_put_failure;
++
++	if (devlink_dpipe_actions_put(table, skb))
++		goto nla_put_failure;
++
++	nla_nest_end(skb, table_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, table_attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
++					    struct genl_info *info)
++{
++	int err;
++
++	if (*pskb) {
++		err = genlmsg_reply(*pskb, info);
++		if (err)
++			return err;
++	}
++	*pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!*pskb)
++		return -ENOMEM;
++	return 0;
++}
++
++static int devlink_dpipe_tables_fill(struct genl_info *info,
++				     enum devlink_command cmd, int flags,
++				     struct list_head *dpipe_tables,
++				     const char *table_name)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_dpipe_table *table;
++	struct nlattr *tables_attr;
++	struct sk_buff *skb = NULL;
++	struct nlmsghdr *nlh;
++	bool incomplete;
++	void *hdr;
++	int i;
++	int err;
++
++	table = list_first_entry(dpipe_tables,
++				 struct devlink_dpipe_table, list);
++start_again:
++	err = devlink_dpipe_send_and_alloc_skb(&skb, info);
++	if (err)
++		return err;
++
++	hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
++			  &devlink_nl_family, NLM_F_MULTI, cmd);
++	if (!hdr) {
++		nlmsg_free(skb);
++		return -EMSGSIZE;
++	}
++
++	if (devlink_nl_put_handle(skb, devlink))
++		goto nla_put_failure;
++	tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
++	if (!tables_attr)
++		goto nla_put_failure;
++
++	i = 0;
++	incomplete = false;
++	list_for_each_entry_from(table, dpipe_tables, list) {
++		if (!table_name) {
++			err = devlink_dpipe_table_put(skb, table);
++			if (err) {
++				if (!i)
++					goto err_table_put;
++				incomplete = true;
++				break;
++			}
++		} else {
++			if (!strcmp(table->name, table_name)) {
++				err = devlink_dpipe_table_put(skb, table);
++				if (err)
++					break;
++			}
++		}
++		i++;
++	}
++
++	nla_nest_end(skb, tables_attr);
++	genlmsg_end(skb, hdr);
++	if (incomplete)
++		goto start_again;
++
++send_done:
++	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
++			NLMSG_DONE, 0, flags | NLM_F_MULTI);
++	if (!nlh) {
++		err = devlink_dpipe_send_and_alloc_skb(&skb, info);
++		if (err)
++			return err;
++		goto send_done;
++	}
++
++	return genlmsg_reply(skb, info);
++
++nla_put_failure:
++	err = -EMSGSIZE;
++err_table_put:
++	nlmsg_free(skb);
++	return err;
++}
++
++static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
++					  struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	const char *table_name =  NULL;
++
++	if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
++		table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
++
++	return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
++					 &devlink->dpipe_table_list,
++					 table_name);
++}
++
++static int devlink_dpipe_value_put(struct sk_buff *skb,
++				   struct devlink_dpipe_value *value)
++{
++	if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
++		    value->value_size, value->value))
++		return -EMSGSIZE;
++	if (value->mask)
++		if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
++			    value->value_size, value->mask))
++			return -EMSGSIZE;
++	if (value->mapping_valid)
++		if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
++				value->mapping_value))
++			return -EMSGSIZE;
++	return 0;
++}
++
++static int devlink_dpipe_action_value_put(struct sk_buff *skb,
++					  struct devlink_dpipe_value *value)
++{
++	if (!value->action)
++		return -EINVAL;
++	if (devlink_dpipe_action_put(skb, value->action))
++		return -EMSGSIZE;
++	if (devlink_dpipe_value_put(skb, value))
++		return -EMSGSIZE;
++	return 0;
++}
++
++static int devlink_dpipe_action_values_put(struct sk_buff *skb,
++					   struct devlink_dpipe_value *values,
++					   unsigned int values_count)
++{
++	struct nlattr *action_attr;
++	int i;
++	int err;
++
++	for (i = 0; i < values_count; i++) {
++		action_attr = nla_nest_start_noflag(skb,
++						    DEVLINK_ATTR_DPIPE_ACTION_VALUE);
++		if (!action_attr)
++			return -EMSGSIZE;
++		err = devlink_dpipe_action_value_put(skb, &values[i]);
++		if (err)
++			goto err_action_value_put;
++		nla_nest_end(skb, action_attr);
++	}
++	return 0;
++
++err_action_value_put:
++	nla_nest_cancel(skb, action_attr);
++	return err;
++}
++
++static int devlink_dpipe_match_value_put(struct sk_buff *skb,
++					 struct devlink_dpipe_value *value)
++{
++	if (!value->match)
++		return -EINVAL;
++	if (devlink_dpipe_match_put(skb, value->match))
++		return -EMSGSIZE;
++	if (devlink_dpipe_value_put(skb, value))
++		return -EMSGSIZE;
++	return 0;
++}
++
++static int devlink_dpipe_match_values_put(struct sk_buff *skb,
++					  struct devlink_dpipe_value *values,
++					  unsigned int values_count)
++{
++	struct nlattr *match_attr;
++	int i;
++	int err;
++
++	for (i = 0; i < values_count; i++) {
++		match_attr = nla_nest_start_noflag(skb,
++						   DEVLINK_ATTR_DPIPE_MATCH_VALUE);
++		if (!match_attr)
++			return -EMSGSIZE;
++		err = devlink_dpipe_match_value_put(skb, &values[i]);
++		if (err)
++			goto err_match_value_put;
++		nla_nest_end(skb, match_attr);
++	}
++	return 0;
++
++err_match_value_put:
++	nla_nest_cancel(skb, match_attr);
++	return err;
++}
++
++static int devlink_dpipe_entry_put(struct sk_buff *skb,
++				   struct devlink_dpipe_entry *entry)
++{
++	struct nlattr *entry_attr, *matches_attr, *actions_attr;
++	int err;
++
++	entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
++	if (!entry_attr)
++		return  -EMSGSIZE;
++
++	if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++	if (entry->counter_valid)
++		if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
++				      entry->counter, DEVLINK_ATTR_PAD))
++			goto nla_put_failure;
++
++	matches_attr = nla_nest_start_noflag(skb,
++					     DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
++	if (!matches_attr)
++		goto nla_put_failure;
++
++	err = devlink_dpipe_match_values_put(skb, entry->match_values,
++					     entry->match_values_count);
++	if (err) {
++		nla_nest_cancel(skb, matches_attr);
++		goto err_match_values_put;
++	}
++	nla_nest_end(skb, matches_attr);
++
++	actions_attr = nla_nest_start_noflag(skb,
++					     DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
++	if (!actions_attr)
++		goto nla_put_failure;
++
++	err = devlink_dpipe_action_values_put(skb, entry->action_values,
++					      entry->action_values_count);
++	if (err) {
++		nla_nest_cancel(skb, actions_attr);
++		goto err_action_values_put;
++	}
++	nla_nest_end(skb, actions_attr);
++
++	nla_nest_end(skb, entry_attr);
++	return 0;
++
++nla_put_failure:
++	err = -EMSGSIZE;
++err_match_values_put:
++err_action_values_put:
++	nla_nest_cancel(skb, entry_attr);
++	return err;
++}
++
++static struct devlink_dpipe_table *
++devlink_dpipe_table_find(struct list_head *dpipe_tables,
++			 const char *table_name, struct devlink *devlink)
++{
++	struct devlink_dpipe_table *table;
++	list_for_each_entry_rcu(table, dpipe_tables, list,
++				lockdep_is_held(&devlink->lock)) {
++		if (!strcmp(table->name, table_name))
++			return table;
++	}
++	return NULL;
++}
++
++int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
++{
++	struct devlink *devlink;
++	int err;
++
++	err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
++					       dump_ctx->info);
++	if (err)
++		return err;
++
++	dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
++				    dump_ctx->info->snd_portid,
++				    dump_ctx->info->snd_seq,
++				    &devlink_nl_family, NLM_F_MULTI,
++				    dump_ctx->cmd);
++	if (!dump_ctx->hdr)
++		goto nla_put_failure;
++
++	devlink = dump_ctx->info->user_ptr[0];
++	if (devlink_nl_put_handle(dump_ctx->skb, devlink))
++		goto nla_put_failure;
++	dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
++					       DEVLINK_ATTR_DPIPE_ENTRIES);
++	if (!dump_ctx->nest)
++		goto nla_put_failure;
++	return 0;
++
++nla_put_failure:
++	nlmsg_free(dump_ctx->skb);
++	return -EMSGSIZE;
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
++
++int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
++				   struct devlink_dpipe_entry *entry)
++{
++	return devlink_dpipe_entry_put(dump_ctx->skb, entry);
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
++
++int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
++{
++	nla_nest_end(dump_ctx->skb, dump_ctx->nest);
++	genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
++
++void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
++
++{
++	unsigned int value_count, value_index;
++	struct devlink_dpipe_value *value;
++
++	value = entry->action_values;
++	value_count = entry->action_values_count;
++	for (value_index = 0; value_index < value_count; value_index++) {
++		kfree(value[value_index].value);
++		kfree(value[value_index].mask);
++	}
++
++	value = entry->match_values;
++	value_count = entry->match_values_count;
++	for (value_index = 0; value_index < value_count; value_index++) {
++		kfree(value[value_index].value);
++		kfree(value[value_index].mask);
++	}
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_entry_clear);
++
++static int devlink_dpipe_entries_fill(struct genl_info *info,
++				      enum devlink_command cmd, int flags,
++				      struct devlink_dpipe_table *table)
++{
++	struct devlink_dpipe_dump_ctx dump_ctx;
++	struct nlmsghdr *nlh;
++	int err;
++
++	dump_ctx.skb = NULL;
++	dump_ctx.cmd = cmd;
++	dump_ctx.info = info;
++
++	err = table->table_ops->entries_dump(table->priv,
++					     table->counters_enabled,
++					     &dump_ctx);
++	if (err)
++		return err;
++
++send_done:
++	nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
++			NLMSG_DONE, 0, flags | NLM_F_MULTI);
++	if (!nlh) {
++		err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
++		if (err)
++			return err;
++		goto send_done;
++	}
++	return genlmsg_reply(dump_ctx.skb, info);
++}
++
++static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
++					    struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_dpipe_table *table;
++	const char *table_name;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME))
++		return -EINVAL;
++
++	table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
++	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
++					 table_name, devlink);
++	if (!table)
++		return -EINVAL;
++
++	if (!table->table_ops->entries_dump)
++		return -EINVAL;
++
++	return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
++					  0, table);
++}
++
++static int devlink_dpipe_fields_put(struct sk_buff *skb,
++				    const struct devlink_dpipe_header *header)
++{
++	struct devlink_dpipe_field *field;
++	struct nlattr *field_attr;
++	int i;
++
++	for (i = 0; i < header->fields_count; i++) {
++		field = &header->fields[i];
++		field_attr = nla_nest_start_noflag(skb,
++						   DEVLINK_ATTR_DPIPE_FIELD);
++		if (!field_attr)
++			return -EMSGSIZE;
++		if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
++		    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
++		    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
++		    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
++			goto nla_put_failure;
++		nla_nest_end(skb, field_attr);
++	}
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, field_attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_dpipe_header_put(struct sk_buff *skb,
++				    struct devlink_dpipe_header *header)
++{
++	struct nlattr *fields_attr, *header_attr;
++	int err;
++
++	header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
++	if (!header_attr)
++		return -EMSGSIZE;
++
++	if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
++	    nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
++	    nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
++		goto nla_put_failure;
++
++	fields_attr = nla_nest_start_noflag(skb,
++					    DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
++	if (!fields_attr)
++		goto nla_put_failure;
++
++	err = devlink_dpipe_fields_put(skb, header);
++	if (err) {
++		nla_nest_cancel(skb, fields_attr);
++		goto nla_put_failure;
++	}
++	nla_nest_end(skb, fields_attr);
++	nla_nest_end(skb, header_attr);
++	return 0;
++
++nla_put_failure:
++	err = -EMSGSIZE;
++	nla_nest_cancel(skb, header_attr);
++	return err;
++}
++
++static int devlink_dpipe_headers_fill(struct genl_info *info,
++				      enum devlink_command cmd, int flags,
++				      struct devlink_dpipe_headers *
++				      dpipe_headers)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct nlattr *headers_attr;
++	struct sk_buff *skb = NULL;
++	struct nlmsghdr *nlh;
++	void *hdr;
++	int i, j;
++	int err;
++
++	i = 0;
++start_again:
++	err = devlink_dpipe_send_and_alloc_skb(&skb, info);
++	if (err)
++		return err;
++
++	hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
++			  &devlink_nl_family, NLM_F_MULTI, cmd);
++	if (!hdr) {
++		nlmsg_free(skb);
++		return -EMSGSIZE;
++	}
++
++	if (devlink_nl_put_handle(skb, devlink))
++		goto nla_put_failure;
++	headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
++	if (!headers_attr)
++		goto nla_put_failure;
++
++	j = 0;
++	for (; i < dpipe_headers->headers_count; i++) {
++		err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
++		if (err) {
++			if (!j)
++				goto err_table_put;
++			break;
++		}
++		j++;
++	}
++	nla_nest_end(skb, headers_attr);
++	genlmsg_end(skb, hdr);
++	if (i != dpipe_headers->headers_count)
++		goto start_again;
++
++send_done:
++	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
++			NLMSG_DONE, 0, flags | NLM_F_MULTI);
++	if (!nlh) {
++		err = devlink_dpipe_send_and_alloc_skb(&skb, info);
++		if (err)
++			return err;
++		goto send_done;
++	}
++	return genlmsg_reply(skb, info);
++
++nla_put_failure:
++	err = -EMSGSIZE;
++err_table_put:
++	nlmsg_free(skb);
++	return err;
++}
++
++static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
++					    struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++
++	if (!devlink->dpipe_headers)
++		return -EOPNOTSUPP;
++	return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
++					  0, devlink->dpipe_headers);
++}
++
++static int devlink_dpipe_table_counters_set(struct devlink *devlink,
++					    const char *table_name,
++					    bool enable)
++{
++	struct devlink_dpipe_table *table;
++
++	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
++					 table_name, devlink);
++	if (!table)
++		return -EINVAL;
++
++	if (table->counter_control_extern)
++		return -EOPNOTSUPP;
++
++	if (!(table->counters_enabled ^ enable))
++		return 0;
++
++	table->counters_enabled = enable;
++	if (table->table_ops->counters_set_update)
++		table->table_ops->counters_set_update(table->priv, enable);
++	return 0;
++}
++
++static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
++						   struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	const char *table_name;
++	bool counters_enable;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME) ||
++	    GENL_REQ_ATTR_CHECK(info,
++				DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED))
++		return -EINVAL;
++
++	table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
++	counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
++
++	return devlink_dpipe_table_counters_set(devlink, table_name,
++						counters_enable);
++}
++
++static struct devlink_resource *
++devlink_resource_find(struct devlink *devlink,
++		      struct devlink_resource *resource, u64 resource_id)
++{
++	struct list_head *resource_list;
++
++	if (resource)
++		resource_list = &resource->resource_list;
++	else
++		resource_list = &devlink->resource_list;
++
++	list_for_each_entry(resource, resource_list, list) {
++		struct devlink_resource *child_resource;
++
++		if (resource->id == resource_id)
++			return resource;
++
++		child_resource = devlink_resource_find(devlink, resource,
++						       resource_id);
++		if (child_resource)
++			return child_resource;
++	}
++	return NULL;
++}
++
++static void
++devlink_resource_validate_children(struct devlink_resource *resource)
++{
++	struct devlink_resource *child_resource;
++	bool size_valid = true;
++	u64 parts_size = 0;
++
++	if (list_empty(&resource->resource_list))
++		goto out;
++
++	list_for_each_entry(child_resource, &resource->resource_list, list)
++		parts_size += child_resource->size_new;
++
++	if (parts_size > resource->size_new)
++		size_valid = false;
++out:
++	resource->size_valid = size_valid;
++}
++
++static int
++devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
++			       struct netlink_ext_ack *extack)
++{
++	u64 reminder;
++	int err = 0;
++
++	if (size > resource->size_params.size_max) {
++		NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
++		err = -EINVAL;
++	}
++
++	if (size < resource->size_params.size_min) {
++		NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
++		err = -EINVAL;
++	}
++
++	div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
++	if (reminder) {
++		NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
++		err = -EINVAL;
++	}
++
++	return err;
++}
++
++static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
++				       struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_resource *resource;
++	u64 resource_id;
++	u64 size;
++	int err;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_ID) ||
++	    GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_SIZE))
++		return -EINVAL;
++	resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
++
++	resource = devlink_resource_find(devlink, NULL, resource_id);
++	if (!resource)
++		return -EINVAL;
++
++	size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
++	err = devlink_resource_validate_size(resource, size, info->extack);
++	if (err)
++		return err;
++
++	resource->size_new = size;
++	devlink_resource_validate_children(resource);
++	if (resource->parent)
++		devlink_resource_validate_children(resource->parent);
++	return 0;
++}
++
++static int
++devlink_resource_size_params_put(struct devlink_resource *resource,
++				 struct sk_buff *skb)
++{
++	struct devlink_resource_size_params *size_params;
++
++	size_params = &resource->size_params;
++	if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
++			      size_params->size_granularity, DEVLINK_ATTR_PAD) ||
++	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
++			      size_params->size_max, DEVLINK_ATTR_PAD) ||
++	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
++			      size_params->size_min, DEVLINK_ATTR_PAD) ||
++	    nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
++		return -EMSGSIZE;
++	return 0;
++}
++
++static int devlink_resource_occ_put(struct devlink_resource *resource,
++				    struct sk_buff *skb)
++{
++	if (!resource->occ_get)
++		return 0;
++	return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
++				 resource->occ_get(resource->occ_get_priv),
++				 DEVLINK_ATTR_PAD);
++}
++
++static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
++				struct devlink_resource *resource)
++{
++	struct devlink_resource *child_resource;
++	struct nlattr *child_resource_attr;
++	struct nlattr *resource_attr;
++
++	resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
++	if (!resource_attr)
++		return -EMSGSIZE;
++
++	if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
++	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
++			      DEVLINK_ATTR_PAD) ||
++	    nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++	if (resource->size != resource->size_new)
++		nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
++				  resource->size_new, DEVLINK_ATTR_PAD);
++	if (devlink_resource_occ_put(resource, skb))
++		goto nla_put_failure;
++	if (devlink_resource_size_params_put(resource, skb))
++		goto nla_put_failure;
++	if (list_empty(&resource->resource_list))
++		goto out;
++
++	if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
++		       resource->size_valid))
++		goto nla_put_failure;
++
++	child_resource_attr = nla_nest_start_noflag(skb,
++						    DEVLINK_ATTR_RESOURCE_LIST);
++	if (!child_resource_attr)
++		goto nla_put_failure;
++
++	list_for_each_entry(child_resource, &resource->resource_list, list) {
++		if (devlink_resource_put(devlink, skb, child_resource))
++			goto resource_put_failure;
++	}
++
++	nla_nest_end(skb, child_resource_attr);
++out:
++	nla_nest_end(skb, resource_attr);
++	return 0;
++
++resource_put_failure:
++	nla_nest_cancel(skb, child_resource_attr);
++nla_put_failure:
++	nla_nest_cancel(skb, resource_attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_resource_fill(struct genl_info *info,
++				 enum devlink_command cmd, int flags)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_resource *resource;
++	struct nlattr *resources_attr;
++	struct sk_buff *skb = NULL;
++	struct nlmsghdr *nlh;
++	bool incomplete;
++	void *hdr;
++	int i;
++	int err;
++
++	resource = list_first_entry(&devlink->resource_list,
++				    struct devlink_resource, list);
++start_again:
++	err = devlink_dpipe_send_and_alloc_skb(&skb, info);
++	if (err)
++		return err;
++
++	hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
++			  &devlink_nl_family, NLM_F_MULTI, cmd);
++	if (!hdr) {
++		nlmsg_free(skb);
++		return -EMSGSIZE;
++	}
++
++	if (devlink_nl_put_handle(skb, devlink))
++		goto nla_put_failure;
++
++	resources_attr = nla_nest_start_noflag(skb,
++					       DEVLINK_ATTR_RESOURCE_LIST);
++	if (!resources_attr)
++		goto nla_put_failure;
++
++	incomplete = false;
++	i = 0;
++	list_for_each_entry_from(resource, &devlink->resource_list, list) {
++		err = devlink_resource_put(devlink, skb, resource);
++		if (err) {
++			if (!i)
++				goto err_resource_put;
++			incomplete = true;
++			break;
++		}
++		i++;
++	}
++	nla_nest_end(skb, resources_attr);
++	genlmsg_end(skb, hdr);
++	if (incomplete)
++		goto start_again;
++send_done:
++	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
++			NLMSG_DONE, 0, flags | NLM_F_MULTI);
++	if (!nlh) {
++		err = devlink_dpipe_send_and_alloc_skb(&skb, info);
++		if (err)
++			return err;
++		goto send_done;
++	}
++	return genlmsg_reply(skb, info);
++
++nla_put_failure:
++	err = -EMSGSIZE;
++err_resource_put:
++	nlmsg_free(skb);
++	return err;
++}
++
++static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++
++	if (list_empty(&devlink->resource_list))
++		return -EOPNOTSUPP;
++
++	return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
++}
++
++static int
++devlink_resources_validate(struct devlink *devlink,
++			   struct devlink_resource *resource,
++			   struct genl_info *info)
++{
++	struct list_head *resource_list;
++	int err = 0;
++
++	if (resource)
++		resource_list = &resource->resource_list;
++	else
++		resource_list = &devlink->resource_list;
++
++	list_for_each_entry(resource, resource_list, list) {
++		if (!resource->size_valid)
++			return -EINVAL;
++		err = devlink_resources_validate(devlink, resource, info);
++		if (err)
++			return err;
++	}
++	return err;
++}
++
++static struct net *devlink_netns_get(struct sk_buff *skb,
++				     struct genl_info *info)
++{
++	struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
++	struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
++	struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
++	struct net *net;
++
++	if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
++		NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
++		return ERR_PTR(-EINVAL);
++	}
++
++	if (netns_pid_attr) {
++		net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
++	} else if (netns_fd_attr) {
++		net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
++	} else if (netns_id_attr) {
++		net = get_net_ns_by_id(sock_net(skb->sk),
++				       nla_get_u32(netns_id_attr));
++		if (!net)
++			net = ERR_PTR(-EINVAL);
++	} else {
++		WARN_ON(1);
++		net = ERR_PTR(-EINVAL);
++	}
++	if (IS_ERR(net)) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
++		return ERR_PTR(-EINVAL);
++	}
++	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
++		put_net(net);
++		return ERR_PTR(-EPERM);
++	}
++	return net;
++}
++
++static void devlink_param_notify(struct devlink *devlink,
++				 unsigned int port_index,
++				 struct devlink_param_item *param_item,
++				 enum devlink_command cmd);
++
++static void devlink_ns_change_notify(struct devlink *devlink,
++				     struct net *dest_net, struct net *curr_net,
++				     bool new)
++{
++	struct devlink_param_item *param_item;
++	enum devlink_command cmd;
++
++	/* Userspace needs to be notified about devlink objects
++	 * removed from original and entering new network namespace.
++	 * The rest of the devlink objects are re-created during
++	 * reload process so the notifications are generated separatelly.
++	 */
++
++	if (!dest_net || net_eq(dest_net, curr_net))
++		return;
++
++	if (new)
++		devlink_notify(devlink, DEVLINK_CMD_NEW);
++
++	cmd = new ? DEVLINK_CMD_PARAM_NEW : DEVLINK_CMD_PARAM_DEL;
++	list_for_each_entry(param_item, &devlink->param_list, list)
++		devlink_param_notify(devlink, 0, param_item, cmd);
++
++	if (!new)
++		devlink_notify(devlink, DEVLINK_CMD_DEL);
++}
++
++static bool devlink_reload_supported(const struct devlink_ops *ops)
++{
++	return ops->reload_down && ops->reload_up;
++}
++
++static void devlink_reload_failed_set(struct devlink *devlink,
++				      bool reload_failed)
++{
++	if (devlink->reload_failed == reload_failed)
++		return;
++	devlink->reload_failed = reload_failed;
++	devlink_notify(devlink, DEVLINK_CMD_NEW);
++}
++
++bool devlink_is_reload_failed(const struct devlink *devlink)
++{
++	return devlink->reload_failed;
++}
++EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
++
++static void
++__devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
++			      enum devlink_reload_limit limit, u32 actions_performed)
++{
++	unsigned long actions = actions_performed;
++	int stat_idx;
++	int action;
++
++	for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
++		stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
++		reload_stats[stat_idx]++;
++	}
++	devlink_notify(devlink, DEVLINK_CMD_NEW);
++}
++
++static void
++devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
++			    u32 actions_performed)
++{
++	__devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
++				      actions_performed);
++}
++
++/**
++ *	devlink_remote_reload_actions_performed - Update devlink on reload actions
++ *	  performed which are not a direct result of devlink reload call.
++ *
++ *	This should be called by a driver after performing reload actions in case it was not
++ *	a result of devlink reload call. For example fw_activate was performed as a result
++ *	of devlink reload triggered fw_activate on another host.
++ *	The motivation for this function is to keep data on reload actions performed on this
++ *	function whether it was done due to direct devlink reload call or not.
++ *
++ *	@devlink: devlink
++ *	@limit: reload limit
++ *	@actions_performed: bitmask of actions performed
++ */
++void devlink_remote_reload_actions_performed(struct devlink *devlink,
++					     enum devlink_reload_limit limit,
++					     u32 actions_performed)
++{
++	if (WARN_ON(!actions_performed ||
++		    actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
++		    actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
++		    limit > DEVLINK_RELOAD_LIMIT_MAX))
++		return;
++
++	__devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
++				      actions_performed);
++}
++EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
++
++static int devlink_reload(struct devlink *devlink, struct net *dest_net,
++			  enum devlink_reload_action action, enum devlink_reload_limit limit,
++			  u32 *actions_performed, struct netlink_ext_ack *extack)
++{
++	u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
++	struct net *curr_net;
++	int err;
++
++	memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
++	       sizeof(remote_reload_stats));
++
++	curr_net = devlink_net(devlink);
++	devlink_ns_change_notify(devlink, dest_net, curr_net, false);
++	err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
++	if (err)
++		return err;
++
++	if (dest_net && !net_eq(dest_net, curr_net))
++		write_pnet(&devlink->_net, dest_net);
++
++	err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
++	devlink_reload_failed_set(devlink, !!err);
++	if (err)
++		return err;
++
++	devlink_ns_change_notify(devlink, dest_net, curr_net, true);
++	WARN_ON(!(*actions_performed & BIT(action)));
++	/* Catch driver on updating the remote action within devlink reload */
++	WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
++		       sizeof(remote_reload_stats)));
++	devlink_reload_stats_update(devlink, limit, *actions_performed);
++	return 0;
++}
++
++static int
++devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
++					enum devlink_command cmd, struct genl_info *info)
++{
++	struct sk_buff *msg;
++	void *hdr;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
++	if (!hdr)
++		goto free_msg;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
++			       actions_performed))
++		goto nla_put_failure;
++	genlmsg_end(msg, hdr);
++
++	return genlmsg_reply(msg, info);
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++free_msg:
++	nlmsg_free(msg);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	enum devlink_reload_action action;
++	enum devlink_reload_limit limit;
++	struct net *dest_net = NULL;
++	u32 actions_performed;
++	int err;
++
++	if (!(devlink->features & DEVLINK_F_RELOAD))
++		return -EOPNOTSUPP;
++
++	err = devlink_resources_validate(devlink, NULL, info);
++	if (err) {
++		NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
++		return err;
++	}
++
++	if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
++		action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
++	else
++		action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
++
++	if (!devlink_reload_action_is_supported(devlink, action)) {
++		NL_SET_ERR_MSG_MOD(info->extack,
++				   "Requested reload action is not supported by the driver");
++		return -EOPNOTSUPP;
++	}
++
++	limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
++	if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
++		struct nla_bitfield32 limits;
++		u32 limits_selected;
++
++		limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
++		limits_selected = limits.value & limits.selector;
++		if (!limits_selected) {
++			NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
++			return -EINVAL;
++		}
++		for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
++			if (limits_selected & BIT(limit))
++				break;
++		/* UAPI enables multiselection, but currently it is not used */
++		if (limits_selected != BIT(limit)) {
++			NL_SET_ERR_MSG_MOD(info->extack,
++					   "Multiselection of limit is not supported");
++			return -EOPNOTSUPP;
++		}
++		if (!devlink_reload_limit_is_supported(devlink, limit)) {
++			NL_SET_ERR_MSG_MOD(info->extack,
++					   "Requested limit is not supported by the driver");
++			return -EOPNOTSUPP;
++		}
++		if (devlink_reload_combination_is_invalid(action, limit)) {
++			NL_SET_ERR_MSG_MOD(info->extack,
++					   "Requested limit is invalid for this action");
++			return -EINVAL;
++		}
++	}
++	if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
++	    info->attrs[DEVLINK_ATTR_NETNS_FD] ||
++	    info->attrs[DEVLINK_ATTR_NETNS_ID]) {
++		dest_net = devlink_netns_get(skb, info);
++		if (IS_ERR(dest_net))
++			return PTR_ERR(dest_net);
++	}
++
++	err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
++
++	if (dest_net)
++		put_net(dest_net);
++
++	if (err)
++		return err;
++	/* For backward compatibility generate reply only if attributes used by user */
++	if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
++		return 0;
++
++	return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
++						       DEVLINK_CMD_RELOAD, info);
++}
++
++static int devlink_nl_flash_update_fill(struct sk_buff *msg,
++					struct devlink *devlink,
++					enum devlink_command cmd,
++					struct devlink_flash_notify *params)
++{
++	void *hdr;
++
++	hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
++		goto out;
++
++	if (params->status_msg &&
++	    nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
++			   params->status_msg))
++		goto nla_put_failure;
++	if (params->component &&
++	    nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
++			   params->component))
++		goto nla_put_failure;
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
++			      params->done, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
++			      params->total, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
++			      params->timeout, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++out:
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static void __devlink_flash_update_notify(struct devlink *devlink,
++					  enum devlink_command cmd,
++					  struct devlink_flash_notify *params)
++{
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
++		cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
++		cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
++
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
++	if (err)
++		goto out_free_msg;
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++	return;
++
++out_free_msg:
++	nlmsg_free(msg);
++}
++
++static void devlink_flash_update_begin_notify(struct devlink *devlink)
++{
++	struct devlink_flash_notify params = {};
++
++	__devlink_flash_update_notify(devlink,
++				      DEVLINK_CMD_FLASH_UPDATE,
++				      &params);
++}
++
++static void devlink_flash_update_end_notify(struct devlink *devlink)
++{
++	struct devlink_flash_notify params = {};
++
++	__devlink_flash_update_notify(devlink,
++				      DEVLINK_CMD_FLASH_UPDATE_END,
++				      &params);
++}
++
++void devlink_flash_update_status_notify(struct devlink *devlink,
++					const char *status_msg,
++					const char *component,
++					unsigned long done,
++					unsigned long total)
++{
++	struct devlink_flash_notify params = {
++		.status_msg = status_msg,
++		.component = component,
++		.done = done,
++		.total = total,
++	};
++
++	__devlink_flash_update_notify(devlink,
++				      DEVLINK_CMD_FLASH_UPDATE_STATUS,
++				      &params);
++}
++EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
++
++void devlink_flash_update_timeout_notify(struct devlink *devlink,
++					 const char *status_msg,
++					 const char *component,
++					 unsigned long timeout)
++{
++	struct devlink_flash_notify params = {
++		.status_msg = status_msg,
++		.component = component,
++		.timeout = timeout,
++	};
++
++	__devlink_flash_update_notify(devlink,
++				      DEVLINK_CMD_FLASH_UPDATE_STATUS,
++				      &params);
++}
++EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
++
++struct devlink_info_req {
++	struct sk_buff *msg;
++	void (*version_cb)(const char *version_name,
++			   enum devlink_info_version_type version_type,
++			   void *version_cb_priv);
++	void *version_cb_priv;
++};
++
++struct devlink_flash_component_lookup_ctx {
++	const char *lookup_name;
++	bool lookup_name_found;
++};
++
++static void
++devlink_flash_component_lookup_cb(const char *version_name,
++				  enum devlink_info_version_type version_type,
++				  void *version_cb_priv)
++{
++	struct devlink_flash_component_lookup_ctx *lookup_ctx = version_cb_priv;
++
++	if (version_type != DEVLINK_INFO_VERSION_TYPE_COMPONENT ||
++	    lookup_ctx->lookup_name_found)
++		return;
++
++	lookup_ctx->lookup_name_found =
++		!strcmp(lookup_ctx->lookup_name, version_name);
++}
++
++static int devlink_flash_component_get(struct devlink *devlink,
++				       struct nlattr *nla_component,
++				       const char **p_component,
++				       struct netlink_ext_ack *extack)
++{
++	struct devlink_flash_component_lookup_ctx lookup_ctx = {};
++	struct devlink_info_req req = {};
++	const char *component;
++	int ret;
++
++	if (!nla_component)
++		return 0;
++
++	component = nla_data(nla_component);
++
++	if (!devlink->ops->info_get) {
++		NL_SET_ERR_MSG_ATTR(extack, nla_component,
++				    "component update is not supported by this device");
++		return -EOPNOTSUPP;
++	}
++
++	lookup_ctx.lookup_name = component;
++	req.version_cb = devlink_flash_component_lookup_cb;
++	req.version_cb_priv = &lookup_ctx;
++
++	ret = devlink->ops->info_get(devlink, &req, NULL);
++	if (ret)
++		return ret;
++
++	if (!lookup_ctx.lookup_name_found) {
++		NL_SET_ERR_MSG_ATTR(extack, nla_component,
++				    "selected component is not supported by this device");
++		return -EINVAL;
++	}
++	*p_component = component;
++	return 0;
++}
++
++static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
++				       struct genl_info *info)
++{
++	struct nlattr *nla_overwrite_mask, *nla_file_name;
++	struct devlink_flash_update_params params = {};
++	struct devlink *devlink = info->user_ptr[0];
++	const char *file_name;
++	u32 supported_params;
++	int ret;
++
++	if (!devlink->ops->flash_update)
++		return -EOPNOTSUPP;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME))
++		return -EINVAL;
++
++	ret = devlink_flash_component_get(devlink,
++					  info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT],
++					  &params.component, info->extack);
++	if (ret)
++		return ret;
++
++	supported_params = devlink->ops->supported_flash_update_params;
++
++	nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
++	if (nla_overwrite_mask) {
++		struct nla_bitfield32 sections;
++
++		if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
++			NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
++					    "overwrite settings are not supported by this device");
++			return -EOPNOTSUPP;
++		}
++		sections = nla_get_bitfield32(nla_overwrite_mask);
++		params.overwrite_mask = sections.value & sections.selector;
++	}
++
++	nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
++	file_name = nla_data(nla_file_name);
++	ret = request_firmware(&params.fw, file_name, devlink->dev);
++	if (ret) {
++		NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name, "failed to locate the requested firmware file");
++		return ret;
++	}
++
++	devlink_flash_update_begin_notify(devlink);
++	ret = devlink->ops->flash_update(devlink, &params, info->extack);
++	devlink_flash_update_end_notify(devlink);
++
++	release_firmware(params.fw);
++
++	return ret;
++}
++
++static int
++devlink_nl_selftests_fill(struct sk_buff *msg, struct devlink *devlink,
++			  u32 portid, u32 seq, int flags,
++			  struct netlink_ext_ack *extack)
++{
++	struct nlattr *selftests;
++	void *hdr;
++	int err;
++	int i;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags,
++			  DEVLINK_CMD_SELFTESTS_GET);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	err = -EMSGSIZE;
++	if (devlink_nl_put_handle(msg, devlink))
++		goto err_cancel_msg;
++
++	selftests = nla_nest_start(msg, DEVLINK_ATTR_SELFTESTS);
++	if (!selftests)
++		goto err_cancel_msg;
++
++	for (i = DEVLINK_ATTR_SELFTEST_ID_UNSPEC + 1;
++	     i <= DEVLINK_ATTR_SELFTEST_ID_MAX; i++) {
++		if (devlink->ops->selftest_check(devlink, i, extack)) {
++			err = nla_put_flag(msg, i);
++			if (err)
++				goto err_cancel_msg;
++		}
++	}
++
++	nla_nest_end(msg, selftests);
++	genlmsg_end(msg, hdr);
++	return 0;
++
++err_cancel_msg:
++	genlmsg_cancel(msg, hdr);
++	return err;
++}
++
++static int devlink_nl_cmd_selftests_get_doit(struct sk_buff *skb,
++					     struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct sk_buff *msg;
++	int err;
++
++	if (!devlink->ops->selftest_check)
++		return -EOPNOTSUPP;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_selftests_fill(msg, devlink, info->snd_portid,
++					info->snd_seq, 0, info->extack);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_selftests_get_dumpit(struct sk_buff *msg,
++					       struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		if (idx < start || !devlink->ops->selftest_check)
++			goto inc;
++
++		devl_lock(devlink);
++		err = devlink_nl_selftests_fill(msg, devlink,
++						NETLINK_CB(cb->skb).portid,
++						cb->nlh->nlmsg_seq, NLM_F_MULTI,
++						cb->extack);
++		devl_unlock(devlink);
++		if (err) {
++			devlink_put(devlink);
++			break;
++		}
++inc:
++		idx++;
++		devlink_put(devlink);
++	}
++
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_selftest_result_put(struct sk_buff *skb, unsigned int id,
++				       enum devlink_selftest_status test_status)
++{
++	struct nlattr *result_attr;
++
++	result_attr = nla_nest_start(skb, DEVLINK_ATTR_SELFTEST_RESULT);
++	if (!result_attr)
++		return -EMSGSIZE;
++
++	if (nla_put_u32(skb, DEVLINK_ATTR_SELFTEST_RESULT_ID, id) ||
++	    nla_put_u8(skb, DEVLINK_ATTR_SELFTEST_RESULT_STATUS,
++		       test_status))
++		goto nla_put_failure;
++
++	nla_nest_end(skb, result_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(skb, result_attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_selftests_run(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct nlattr *tb[DEVLINK_ATTR_SELFTEST_ID_MAX + 1];
++	struct devlink *devlink = info->user_ptr[0];
++	struct nlattr *attrs, *selftests;
++	struct sk_buff *msg;
++	void *hdr;
++	int err;
++	int i;
++
++	if (!devlink->ops->selftest_run || !devlink->ops->selftest_check)
++		return -EOPNOTSUPP;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SELFTESTS))
++		return -EINVAL;
++
++	attrs = info->attrs[DEVLINK_ATTR_SELFTESTS];
++
++	err = nla_parse_nested(tb, DEVLINK_ATTR_SELFTEST_ID_MAX, attrs,
++			       devlink_selftest_nl_policy, info->extack);
++	if (err < 0)
++		return err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = -EMSGSIZE;
++	hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
++			  &devlink_nl_family, 0, DEVLINK_CMD_SELFTESTS_RUN);
++	if (!hdr)
++		goto free_msg;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto genlmsg_cancel;
++
++	selftests = nla_nest_start(msg, DEVLINK_ATTR_SELFTESTS);
++	if (!selftests)
++		goto genlmsg_cancel;
++
++	for (i = DEVLINK_ATTR_SELFTEST_ID_UNSPEC + 1;
++	     i <= DEVLINK_ATTR_SELFTEST_ID_MAX; i++) {
++		enum devlink_selftest_status test_status;
++
++		if (nla_get_flag(tb[i])) {
++			if (!devlink->ops->selftest_check(devlink, i,
++							  info->extack)) {
++				if (devlink_selftest_result_put(msg, i,
++								DEVLINK_SELFTEST_STATUS_SKIP))
++					goto selftests_nest_cancel;
++				continue;
++			}
++
++			test_status = devlink->ops->selftest_run(devlink, i,
++								 info->extack);
++			if (devlink_selftest_result_put(msg, i, test_status))
++				goto selftests_nest_cancel;
++		}
++	}
++
++	nla_nest_end(msg, selftests);
++	genlmsg_end(msg, hdr);
++	return genlmsg_reply(msg, info);
++
++selftests_nest_cancel:
++	nla_nest_cancel(msg, selftests);
++genlmsg_cancel:
++	genlmsg_cancel(msg, hdr);
++free_msg:
++	nlmsg_free(msg);
++	return err;
++}
++
++static const struct devlink_param devlink_param_generic[] = {
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
++		.name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
++		.type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
++		.name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
++		.type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
++		.name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
++		.type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
++		.name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
++		.type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
++		.name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
++		.type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
++		.name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
++		.type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
++		.name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
++		.type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
++		.name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
++		.type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP,
++		.name = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME,
++		.type = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
++		.name = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_NAME,
++		.type = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_TYPE,
++	},
++	{
++		.id = DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
++		.name = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME,
++		.type = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE,
++	},
++};
++
++static int devlink_param_generic_verify(const struct devlink_param *param)
++{
++	/* verify it match generic parameter by id and name */
++	if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
++		return -EINVAL;
++	if (strcmp(param->name, devlink_param_generic[param->id].name))
++		return -ENOENT;
++
++	WARN_ON(param->type != devlink_param_generic[param->id].type);
++
++	return 0;
++}
++
++static int devlink_param_driver_verify(const struct devlink_param *param)
++{
++	int i;
++
++	if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
++		return -EINVAL;
++	/* verify no such name in generic params */
++	for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
++		if (!strcmp(param->name, devlink_param_generic[i].name))
++			return -EEXIST;
++
++	return 0;
++}
++
++static struct devlink_param_item *
++devlink_param_find_by_name(struct list_head *param_list,
++			   const char *param_name)
++{
++	struct devlink_param_item *param_item;
++
++	list_for_each_entry(param_item, param_list, list)
++		if (!strcmp(param_item->param->name, param_name))
++			return param_item;
++	return NULL;
++}
++
++static struct devlink_param_item *
++devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
++{
++	struct devlink_param_item *param_item;
++
++	list_for_each_entry(param_item, param_list, list)
++		if (param_item->param->id == param_id)
++			return param_item;
++	return NULL;
++}
++
++static bool
++devlink_param_cmode_is_supported(const struct devlink_param *param,
++				 enum devlink_param_cmode cmode)
++{
++	return test_bit(cmode, &param->supported_cmodes);
++}
++
++static int devlink_param_get(struct devlink *devlink,
++			     const struct devlink_param *param,
++			     struct devlink_param_gset_ctx *ctx)
++{
++	if (!param->get || devlink->reload_failed)
++		return -EOPNOTSUPP;
++	return param->get(devlink, param->id, ctx);
++}
++
++static int devlink_param_set(struct devlink *devlink,
++			     const struct devlink_param *param,
++			     struct devlink_param_gset_ctx *ctx)
++{
++	if (!param->set || devlink->reload_failed)
++		return -EOPNOTSUPP;
++	return param->set(devlink, param->id, ctx);
++}
++
++static int
++devlink_param_type_to_nla_type(enum devlink_param_type param_type)
++{
++	switch (param_type) {
++	case DEVLINK_PARAM_TYPE_U8:
++		return NLA_U8;
++	case DEVLINK_PARAM_TYPE_U16:
++		return NLA_U16;
++	case DEVLINK_PARAM_TYPE_U32:
++		return NLA_U32;
++	case DEVLINK_PARAM_TYPE_STRING:
++		return NLA_STRING;
++	case DEVLINK_PARAM_TYPE_BOOL:
++		return NLA_FLAG;
++	default:
++		return -EINVAL;
++	}
++}
++
++static int
++devlink_nl_param_value_fill_one(struct sk_buff *msg,
++				enum devlink_param_type type,
++				enum devlink_param_cmode cmode,
++				union devlink_param_value val)
++{
++	struct nlattr *param_value_attr;
++
++	param_value_attr = nla_nest_start_noflag(msg,
++						 DEVLINK_ATTR_PARAM_VALUE);
++	if (!param_value_attr)
++		goto nla_put_failure;
++
++	if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
++		goto value_nest_cancel;
++
++	switch (type) {
++	case DEVLINK_PARAM_TYPE_U8:
++		if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
++			goto value_nest_cancel;
++		break;
++	case DEVLINK_PARAM_TYPE_U16:
++		if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
++			goto value_nest_cancel;
++		break;
++	case DEVLINK_PARAM_TYPE_U32:
++		if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
++			goto value_nest_cancel;
++		break;
++	case DEVLINK_PARAM_TYPE_STRING:
++		if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
++				   val.vstr))
++			goto value_nest_cancel;
++		break;
++	case DEVLINK_PARAM_TYPE_BOOL:
++		if (val.vbool &&
++		    nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
++			goto value_nest_cancel;
++		break;
++	}
++
++	nla_nest_end(msg, param_value_attr);
++	return 0;
++
++value_nest_cancel:
++	nla_nest_cancel(msg, param_value_attr);
++nla_put_failure:
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
++				 unsigned int port_index,
++				 struct devlink_param_item *param_item,
++				 enum devlink_command cmd,
++				 u32 portid, u32 seq, int flags)
++{
++	union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
++	bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
++	const struct devlink_param *param = param_item->param;
++	struct devlink_param_gset_ctx ctx;
++	struct nlattr *param_values_list;
++	struct nlattr *param_attr;
++	int nla_type;
++	void *hdr;
++	int err;
++	int i;
++
++	/* Get value from driver part to driverinit configuration mode */
++	for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
++		if (!devlink_param_cmode_is_supported(param, i))
++			continue;
++		if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
++			if (!param_item->driverinit_value_valid)
++				return -EOPNOTSUPP;
++			param_value[i] = param_item->driverinit_value;
++		} else {
++			ctx.cmode = i;
++			err = devlink_param_get(devlink, param, &ctx);
++			if (err)
++				return err;
++			param_value[i] = ctx.val;
++		}
++		param_value_set[i] = true;
++	}
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto genlmsg_cancel;
++
++	if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
++	    cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
++	    cmd == DEVLINK_CMD_PORT_PARAM_DEL)
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
++			goto genlmsg_cancel;
++
++	param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
++	if (!param_attr)
++		goto genlmsg_cancel;
++	if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
++		goto param_nest_cancel;
++	if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
++		goto param_nest_cancel;
++
++	nla_type = devlink_param_type_to_nla_type(param->type);
++	if (nla_type < 0)
++		goto param_nest_cancel;
++	if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
++		goto param_nest_cancel;
++
++	param_values_list = nla_nest_start_noflag(msg,
++						  DEVLINK_ATTR_PARAM_VALUES_LIST);
++	if (!param_values_list)
++		goto param_nest_cancel;
++
++	for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
++		if (!param_value_set[i])
++			continue;
++		err = devlink_nl_param_value_fill_one(msg, param->type,
++						      i, param_value[i]);
++		if (err)
++			goto values_list_nest_cancel;
++	}
++
++	nla_nest_end(msg, param_values_list);
++	nla_nest_end(msg, param_attr);
++	genlmsg_end(msg, hdr);
++	return 0;
++
++values_list_nest_cancel:
++	nla_nest_end(msg, param_values_list);
++param_nest_cancel:
++	nla_nest_cancel(msg, param_attr);
++genlmsg_cancel:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static void devlink_param_notify(struct devlink *devlink,
++				 unsigned int port_index,
++				 struct devlink_param_item *param_item,
++				 enum devlink_command cmd)
++{
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
++		cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
++		cmd != DEVLINK_CMD_PORT_PARAM_DEL);
++	ASSERT_DEVLINK_REGISTERED(devlink);
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++	err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
++				    0, 0, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
++					   struct netlink_callback *cb)
++{
++	struct devlink_param_item *param_item;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(param_item, &devlink->param_list, list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_param_fill(msg, devlink, 0, param_item,
++						    DEVLINK_CMD_PARAM_GET,
++						    NETLINK_CB(cb->skb).portid,
++						    cb->nlh->nlmsg_seq,
++						    NLM_F_MULTI);
++			if (err == -EOPNOTSUPP) {
++				err = 0;
++			} else if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int
++devlink_param_type_get_from_info(struct genl_info *info,
++				 enum devlink_param_type *param_type)
++{
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_TYPE))
++		return -EINVAL;
++
++	switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
++	case NLA_U8:
++		*param_type = DEVLINK_PARAM_TYPE_U8;
++		break;
++	case NLA_U16:
++		*param_type = DEVLINK_PARAM_TYPE_U16;
++		break;
++	case NLA_U32:
++		*param_type = DEVLINK_PARAM_TYPE_U32;
++		break;
++	case NLA_STRING:
++		*param_type = DEVLINK_PARAM_TYPE_STRING;
++		break;
++	case NLA_FLAG:
++		*param_type = DEVLINK_PARAM_TYPE_BOOL;
++		break;
++	default:
++		return -EINVAL;
++	}
++
++	return 0;
++}
++
++static int
++devlink_param_value_get_from_info(const struct devlink_param *param,
++				  struct genl_info *info,
++				  union devlink_param_value *value)
++{
++	struct nlattr *param_data;
++	int len;
++
++	param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
++
++	if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
++		return -EINVAL;
++
++	switch (param->type) {
++	case DEVLINK_PARAM_TYPE_U8:
++		if (nla_len(param_data) != sizeof(u8))
++			return -EINVAL;
++		value->vu8 = nla_get_u8(param_data);
++		break;
++	case DEVLINK_PARAM_TYPE_U16:
++		if (nla_len(param_data) != sizeof(u16))
++			return -EINVAL;
++		value->vu16 = nla_get_u16(param_data);
++		break;
++	case DEVLINK_PARAM_TYPE_U32:
++		if (nla_len(param_data) != sizeof(u32))
++			return -EINVAL;
++		value->vu32 = nla_get_u32(param_data);
++		break;
++	case DEVLINK_PARAM_TYPE_STRING:
++		len = strnlen(nla_data(param_data), nla_len(param_data));
++		if (len == nla_len(param_data) ||
++		    len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
++			return -EINVAL;
++		strcpy(value->vstr, nla_data(param_data));
++		break;
++	case DEVLINK_PARAM_TYPE_BOOL:
++		if (param_data && nla_len(param_data))
++			return -EINVAL;
++		value->vbool = nla_get_flag(param_data);
++		break;
++	}
++	return 0;
++}
++
++static struct devlink_param_item *
++devlink_param_get_from_info(struct list_head *param_list,
++			    struct genl_info *info)
++{
++	char *param_name;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_NAME))
++		return NULL;
++
++	param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
++	return devlink_param_find_by_name(param_list, param_name);
++}
++
++static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
++					 struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_param_item *param_item;
++	struct sk_buff *msg;
++	int err;
++
++	param_item = devlink_param_get_from_info(&devlink->param_list, info);
++	if (!param_item)
++		return -EINVAL;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_param_fill(msg, devlink, 0, param_item,
++				    DEVLINK_CMD_PARAM_GET,
++				    info->snd_portid, info->snd_seq, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
++					   unsigned int port_index,
++					   struct list_head *param_list,
++					   struct genl_info *info,
++					   enum devlink_command cmd)
++{
++	enum devlink_param_type param_type;
++	struct devlink_param_gset_ctx ctx;
++	enum devlink_param_cmode cmode;
++	struct devlink_param_item *param_item;
++	const struct devlink_param *param;
++	union devlink_param_value value;
++	int err = 0;
++
++	param_item = devlink_param_get_from_info(param_list, info);
++	if (!param_item)
++		return -EINVAL;
++	param = param_item->param;
++	err = devlink_param_type_get_from_info(info, &param_type);
++	if (err)
++		return err;
++	if (param_type != param->type)
++		return -EINVAL;
++	err = devlink_param_value_get_from_info(param, info, &value);
++	if (err)
++		return err;
++	if (param->validate) {
++		err = param->validate(devlink, param->id, value, info->extack);
++		if (err)
++			return err;
++	}
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_VALUE_CMODE))
++		return -EINVAL;
++	cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
++	if (!devlink_param_cmode_is_supported(param, cmode))
++		return -EOPNOTSUPP;
++
++	if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
++		if (param->type == DEVLINK_PARAM_TYPE_STRING)
++			strcpy(param_item->driverinit_value.vstr, value.vstr);
++		else
++			param_item->driverinit_value = value;
++		param_item->driverinit_value_valid = true;
++	} else {
++		if (!param->set)
++			return -EOPNOTSUPP;
++		ctx.val = value;
++		ctx.cmode = cmode;
++		err = devlink_param_set(devlink, param, &ctx);
++		if (err)
++			return err;
++	}
++
++	devlink_param_notify(devlink, port_index, param_item, cmd);
++	return 0;
++}
++
++static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
++					 struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++
++	return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
++					       info, DEVLINK_CMD_PARAM_NEW);
++}
++
++static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
++						struct netlink_callback *cb)
++{
++	NL_SET_ERR_MSG_MOD(cb->extack, "Port params are not supported");
++	return msg->len;
++}
++
++static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
++					      struct genl_info *info)
++{
++	NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported");
++	return -EINVAL;
++}
++
++static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
++					      struct genl_info *info)
++{
++	NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported");
++	return -EINVAL;
++}
++
++static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
++					     struct devlink *devlink,
++					     struct devlink_snapshot *snapshot)
++{
++	struct nlattr *snap_attr;
++	int err;
++
++	snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
++	if (!snap_attr)
++		return -EINVAL;
++
++	err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
++	if (err)
++		goto nla_put_failure;
++
++	nla_nest_end(msg, snap_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, snap_attr);
++	return err;
++}
++
++static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
++					      struct devlink *devlink,
++					      struct devlink_region *region)
++{
++	struct devlink_snapshot *snapshot;
++	struct nlattr *snapshots_attr;
++	int err;
++
++	snapshots_attr = nla_nest_start_noflag(msg,
++					       DEVLINK_ATTR_REGION_SNAPSHOTS);
++	if (!snapshots_attr)
++		return -EINVAL;
++
++	list_for_each_entry(snapshot, &region->snapshot_list, list) {
++		err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
++		if (err)
++			goto nla_put_failure;
++	}
++
++	nla_nest_end(msg, snapshots_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, snapshots_attr);
++	return err;
++}
++
++static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
++				  enum devlink_command cmd, u32 portid,
++				  u32 seq, int flags,
++				  struct devlink_region *region)
++{
++	void *hdr;
++	int err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	err = devlink_nl_put_handle(msg, devlink);
++	if (err)
++		goto nla_put_failure;
++
++	if (region->port) {
++		err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
++				  region->port->index);
++		if (err)
++			goto nla_put_failure;
++	}
++
++	err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
++	if (err)
++		goto nla_put_failure;
++
++	err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
++				region->size,
++				DEVLINK_ATTR_PAD);
++	if (err)
++		goto nla_put_failure;
++
++	err = nla_put_u32(msg, DEVLINK_ATTR_REGION_MAX_SNAPSHOTS,
++			  region->max_snapshots);
++	if (err)
++		goto nla_put_failure;
++
++	err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
++	if (err)
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return err;
++}
++
++static struct sk_buff *
++devlink_nl_region_notify_build(struct devlink_region *region,
++			       struct devlink_snapshot *snapshot,
++			       enum devlink_command cmd, u32 portid, u32 seq)
++{
++	struct devlink *devlink = region->devlink;
++	struct sk_buff *msg;
++	void *hdr;
++	int err;
++
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return ERR_PTR(-ENOMEM);
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
++	if (!hdr) {
++		err = -EMSGSIZE;
++		goto out_free_msg;
++	}
++
++	err = devlink_nl_put_handle(msg, devlink);
++	if (err)
++		goto out_cancel_msg;
++
++	if (region->port) {
++		err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
++				  region->port->index);
++		if (err)
++			goto out_cancel_msg;
++	}
++
++	err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
++			     region->ops->name);
++	if (err)
++		goto out_cancel_msg;
++
++	if (snapshot) {
++		err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
++				  snapshot->id);
++		if (err)
++			goto out_cancel_msg;
++	} else {
++		err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
++					region->size, DEVLINK_ATTR_PAD);
++		if (err)
++			goto out_cancel_msg;
++	}
++	genlmsg_end(msg, hdr);
++
++	return msg;
++
++out_cancel_msg:
++	genlmsg_cancel(msg, hdr);
++out_free_msg:
++	nlmsg_free(msg);
++	return ERR_PTR(err);
++}
++
++static void devlink_nl_region_notify(struct devlink_region *region,
++				     struct devlink_snapshot *snapshot,
++				     enum devlink_command cmd)
++{
++	struct devlink *devlink = region->devlink;
++	struct sk_buff *msg;
++
++	WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
++	if (IS_ERR(msg))
++		return;
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
++				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++/**
++ * __devlink_snapshot_id_increment - Increment number of snapshots using an id
++ *	@devlink: devlink instance
++ *	@id: the snapshot id
++ *
++ *	Track when a new snapshot begins using an id. Load the count for the
++ *	given id from the snapshot xarray, increment it, and store it back.
++ *
++ *	Called when a new snapshot is created with the given id.
++ *
++ *	The id *must* have been previously allocated by
++ *	devlink_region_snapshot_id_get().
++ *
++ *	Returns 0 on success, or an error on failure.
++ */
++static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
++{
++	unsigned long count;
++	void *p;
++	int err;
++
++	xa_lock(&devlink->snapshot_ids);
++	p = xa_load(&devlink->snapshot_ids, id);
++	if (WARN_ON(!p)) {
++		err = -EINVAL;
++		goto unlock;
++	}
++
++	if (WARN_ON(!xa_is_value(p))) {
++		err = -EINVAL;
++		goto unlock;
++	}
++
++	count = xa_to_value(p);
++	count++;
++
++	err = xa_err(__xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
++				GFP_ATOMIC));
++unlock:
++	xa_unlock(&devlink->snapshot_ids);
++	return err;
++}
++
++/**
++ * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
++ *	@devlink: devlink instance
++ *	@id: the snapshot id
++ *
++ *	Track when a snapshot is deleted and stops using an id. Load the count
++ *	for the given id from the snapshot xarray, decrement it, and store it
++ *	back.
++ *
++ *	If the count reaches zero, erase this id from the xarray, freeing it
++ *	up for future re-use by devlink_region_snapshot_id_get().
++ *
++ *	Called when a snapshot using the given id is deleted, and when the
++ *	initial allocator of the id is finished using it.
++ */
++static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
++{
++	unsigned long count;
++	void *p;
++
++	xa_lock(&devlink->snapshot_ids);
++	p = xa_load(&devlink->snapshot_ids, id);
++	if (WARN_ON(!p))
++		goto unlock;
++
++	if (WARN_ON(!xa_is_value(p)))
++		goto unlock;
++
++	count = xa_to_value(p);
++
++	if (count > 1) {
++		count--;
++		__xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
++			   GFP_ATOMIC);
++	} else {
++		/* If this was the last user, we can erase this id */
++		__xa_erase(&devlink->snapshot_ids, id);
++	}
++unlock:
++	xa_unlock(&devlink->snapshot_ids);
++}
++
++/**
++ *	__devlink_snapshot_id_insert - Insert a specific snapshot ID
++ *	@devlink: devlink instance
++ *	@id: the snapshot id
++ *
++ *	Mark the given snapshot id as used by inserting a zero value into the
++ *	snapshot xarray.
++ *
++ *	This must be called while holding the devlink instance lock. Unlike
++ *	devlink_snapshot_id_get, the initial reference count is zero, not one.
++ *	It is expected that the id will immediately be used before
++ *	releasing the devlink instance lock.
++ *
++ *	Returns zero on success, or an error code if the snapshot id could not
++ *	be inserted.
++ */
++static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
++{
++	int err;
++
++	xa_lock(&devlink->snapshot_ids);
++	if (xa_load(&devlink->snapshot_ids, id)) {
++		xa_unlock(&devlink->snapshot_ids);
++		return -EEXIST;
++	}
++	err = xa_err(__xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
++				GFP_ATOMIC));
++	xa_unlock(&devlink->snapshot_ids);
++	return err;
++}
++
++/**
++ *	__devlink_region_snapshot_id_get - get snapshot ID
++ *	@devlink: devlink instance
++ *	@id: storage to return snapshot id
++ *
++ *	Allocates a new snapshot id. Returns zero on success, or a negative
++ *	error on failure. Must be called while holding the devlink instance
++ *	lock.
++ *
++ *	Snapshot IDs are tracked using an xarray which stores the number of
++ *	users of the snapshot id.
++ *
++ *	Note that the caller of this function counts as a 'user', in order to
++ *	avoid race conditions. The caller must release its hold on the
++ *	snapshot by using devlink_region_snapshot_id_put.
++ */
++static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
++{
++	return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
++			xa_limit_32b, GFP_KERNEL);
++}
++
++/**
++ *	__devlink_region_snapshot_create - create a new snapshot
++ *	This will add a new snapshot of a region. The snapshot
++ *	will be stored on the region struct and can be accessed
++ *	from devlink. This is useful for future analyses of snapshots.
++ *	Multiple snapshots can be created on a region.
++ *	The @snapshot_id should be obtained using the getter function.
++ *
++ *	Must be called only while holding the region snapshot lock.
++ *
++ *	@region: devlink region of the snapshot
++ *	@data: snapshot data
++ *	@snapshot_id: snapshot id to be created
++ */
++static int
++__devlink_region_snapshot_create(struct devlink_region *region,
++				 u8 *data, u32 snapshot_id)
++{
++	struct devlink *devlink = region->devlink;
++	struct devlink_snapshot *snapshot;
++	int err;
++
++	lockdep_assert_held(&region->snapshot_lock);
++
++	/* check if region can hold one more snapshot */
++	if (region->cur_snapshots == region->max_snapshots)
++		return -ENOSPC;
++
++	if (devlink_region_snapshot_get_by_id(region, snapshot_id))
++		return -EEXIST;
++
++	snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
++	if (!snapshot)
++		return -ENOMEM;
++
++	err = __devlink_snapshot_id_increment(devlink, snapshot_id);
++	if (err)
++		goto err_snapshot_id_increment;
++
++	snapshot->id = snapshot_id;
++	snapshot->region = region;
++	snapshot->data = data;
++
++	list_add_tail(&snapshot->list, &region->snapshot_list);
++
++	region->cur_snapshots++;
++
++	devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
++	return 0;
++
++err_snapshot_id_increment:
++	kfree(snapshot);
++	return err;
++}
++
++static void devlink_region_snapshot_del(struct devlink_region *region,
++					struct devlink_snapshot *snapshot)
++{
++	struct devlink *devlink = region->devlink;
++
++	lockdep_assert_held(&region->snapshot_lock);
++
++	devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
++	region->cur_snapshots--;
++	list_del(&snapshot->list);
++	region->ops->destructor(snapshot->data);
++	__devlink_snapshot_id_decrement(devlink, snapshot->id);
++	kfree(snapshot);
++}
++
++static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
++					  struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_port *port = NULL;
++	struct devlink_region *region;
++	const char *region_name;
++	struct sk_buff *msg;
++	unsigned int index;
++	int err;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME))
++		return -EINVAL;
++
++	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
++		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
++
++		port = devlink_port_get_by_index(devlink, index);
++		if (!port)
++			return -ENODEV;
++	}
++
++	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
++	if (port)
++		region = devlink_port_region_get_by_name(port, region_name);
++	else
++		region = devlink_region_get_by_name(devlink, region_name);
++
++	if (!region)
++		return -EINVAL;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
++				     info->snd_portid, info->snd_seq, 0,
++				     region);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
++						 struct netlink_callback *cb,
++						 struct devlink_port *port,
++						 int *idx,
++						 int start)
++{
++	struct devlink_region *region;
++	int err = 0;
++
++	list_for_each_entry(region, &port->region_list, list) {
++		if (*idx < start) {
++			(*idx)++;
++			continue;
++		}
++		err = devlink_nl_region_fill(msg, port->devlink,
++					     DEVLINK_CMD_REGION_GET,
++					     NETLINK_CB(cb->skb).portid,
++					     cb->nlh->nlmsg_seq,
++					     NLM_F_MULTI, region);
++		if (err)
++			goto out;
++		(*idx)++;
++	}
++
++out:
++	return err;
++}
++
++static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
++						    struct netlink_callback *cb,
++						    struct devlink *devlink,
++						    int *idx,
++						    int start)
++{
++	struct devlink_region *region;
++	struct devlink_port *port;
++	int err = 0;
++
++	devl_lock(devlink);
++	list_for_each_entry(region, &devlink->region_list, list) {
++		if (*idx < start) {
++			(*idx)++;
++			continue;
++		}
++		err = devlink_nl_region_fill(msg, devlink,
++					     DEVLINK_CMD_REGION_GET,
++					     NETLINK_CB(cb->skb).portid,
++					     cb->nlh->nlmsg_seq,
++					     NLM_F_MULTI, region);
++		if (err)
++			goto out;
++		(*idx)++;
++	}
++
++	list_for_each_entry(port, &devlink->port_list, list) {
++		err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
++							    start);
++		if (err)
++			goto out;
++	}
++
++out:
++	devl_unlock(devlink);
++	return err;
++}
++
++static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
++					    struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
++							       &idx, start);
++		devlink_put(devlink);
++		if (err)
++			goto out;
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int devlink_nl_cmd_region_del(struct sk_buff *skb,
++				     struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_snapshot *snapshot;
++	struct devlink_port *port = NULL;
++	struct devlink_region *region;
++	const char *region_name;
++	unsigned int index;
++	u32 snapshot_id;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME) ||
++	    GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_SNAPSHOT_ID))
++		return -EINVAL;
++
++	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
++	snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
++
++	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
++		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
++
++		port = devlink_port_get_by_index(devlink, index);
++		if (!port)
++			return -ENODEV;
++	}
++
++	if (port)
++		region = devlink_port_region_get_by_name(port, region_name);
++	else
++		region = devlink_region_get_by_name(devlink, region_name);
++
++	if (!region)
++		return -EINVAL;
++
++	mutex_lock(&region->snapshot_lock);
++	snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
++	if (!snapshot) {
++		mutex_unlock(&region->snapshot_lock);
++		return -EINVAL;
++	}
++
++	devlink_region_snapshot_del(region, snapshot);
++	mutex_unlock(&region->snapshot_lock);
++	return 0;
++}
++
++static int
++devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_snapshot *snapshot;
++	struct devlink_port *port = NULL;
++	struct nlattr *snapshot_id_attr;
++	struct devlink_region *region;
++	const char *region_name;
++	unsigned int index;
++	u32 snapshot_id;
++	u8 *data;
++	int err;
++
++	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME)) {
++		NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
++		return -EINVAL;
++	}
++
++	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
++
++	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
++		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
++
++		port = devlink_port_get_by_index(devlink, index);
++		if (!port)
++			return -ENODEV;
++	}
++
++	if (port)
++		region = devlink_port_region_get_by_name(port, region_name);
++	else
++		region = devlink_region_get_by_name(devlink, region_name);
++
++	if (!region) {
++		NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
++		return -EINVAL;
++	}
++
++	if (!region->ops->snapshot) {
++		NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
++		return -EOPNOTSUPP;
++	}
++
++	mutex_lock(&region->snapshot_lock);
++
++	if (region->cur_snapshots == region->max_snapshots) {
++		NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
++		err = -ENOSPC;
++		goto unlock;
++	}
++
++	snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
++	if (snapshot_id_attr) {
++		snapshot_id = nla_get_u32(snapshot_id_attr);
++
++		if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
++			NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
++			err = -EEXIST;
++			goto unlock;
++		}
++
++		err = __devlink_snapshot_id_insert(devlink, snapshot_id);
++		if (err)
++			goto unlock;
++	} else {
++		err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
++		if (err) {
++			NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
++			goto unlock;
++		}
++	}
++
++	if (port)
++		err = region->port_ops->snapshot(port, region->port_ops,
++						 info->extack, &data);
++	else
++		err = region->ops->snapshot(devlink, region->ops,
++					    info->extack, &data);
++	if (err)
++		goto err_snapshot_capture;
++
++	err = __devlink_region_snapshot_create(region, data, snapshot_id);
++	if (err)
++		goto err_snapshot_create;
++
++	if (!snapshot_id_attr) {
++		struct sk_buff *msg;
++
++		snapshot = devlink_region_snapshot_get_by_id(region,
++							     snapshot_id);
++		if (WARN_ON(!snapshot)) {
++			err = -EINVAL;
++			goto unlock;
++		}
++
++		msg = devlink_nl_region_notify_build(region, snapshot,
++						     DEVLINK_CMD_REGION_NEW,
++						     info->snd_portid,
++						     info->snd_seq);
++		err = PTR_ERR_OR_ZERO(msg);
++		if (err)
++			goto err_notify;
++
++		err = genlmsg_reply(msg, info);
++		if (err)
++			goto err_notify;
++	}
++
++	mutex_unlock(&region->snapshot_lock);
++	return 0;
++
++err_snapshot_create:
++	region->ops->destructor(data);
++err_snapshot_capture:
++	__devlink_snapshot_id_decrement(devlink, snapshot_id);
++	mutex_unlock(&region->snapshot_lock);
++	return err;
++
++err_notify:
++	devlink_region_snapshot_del(region, snapshot);
++unlock:
++	mutex_unlock(&region->snapshot_lock);
++	return err;
++}
++
++static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
++						 struct devlink *devlink,
++						 u8 *chunk, u32 chunk_size,
++						 u64 addr)
++{
++	struct nlattr *chunk_attr;
++	int err;
++
++	chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
++	if (!chunk_attr)
++		return -EINVAL;
++
++	err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
++	if (err)
++		goto nla_put_failure;
++
++	err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
++				DEVLINK_ATTR_PAD);
++	if (err)
++		goto nla_put_failure;
++
++	nla_nest_end(msg, chunk_attr);
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, chunk_attr);
++	return err;
++}
++
++#define DEVLINK_REGION_READ_CHUNK_SIZE 256
++
++static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
++						struct devlink *devlink,
++						struct devlink_region *region,
++						struct nlattr **attrs,
++						u64 start_offset,
++						u64 end_offset,
++						u64 *new_offset)
++{
++	struct devlink_snapshot *snapshot;
++	u64 curr_offset = start_offset;
++	u32 snapshot_id;
++	int err = 0;
++
++	*new_offset = start_offset;
++
++	snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
++	snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
++	if (!snapshot)
++		return -EINVAL;
++
++	while (curr_offset < end_offset) {
++		u32 data_size;
++		u8 *data;
++
++		if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
++			data_size = end_offset - curr_offset;
++		else
++			data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
++
++		data = &snapshot->data[curr_offset];
++		err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
++							    data, data_size,
++							    curr_offset);
++		if (err)
++			break;
++
++		curr_offset += data_size;
++	}
++	*new_offset = curr_offset;
++
++	return err;
++}
++
++static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
++					     struct netlink_callback *cb)
++{
++	const struct genl_dumpit_info *info = genl_dumpit_info(cb);
++	u64 ret_offset, start_offset, end_offset = U64_MAX;
++	struct nlattr **attrs = info->attrs;
++	struct devlink_port *port = NULL;
++	struct devlink_region *region;
++	struct nlattr *chunks_attr;
++	const char *region_name;
++	struct devlink *devlink;
++	unsigned int index;
++	void *hdr;
++	int err;
++
++	start_offset = *((u64 *)&cb->args[0]);
++
++	devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
++	if (IS_ERR(devlink))
++		return PTR_ERR(devlink);
++
++	devl_lock(devlink);
++
++	if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
++	    !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
++		err = -EINVAL;
++		goto out_unlock;
++	}
++
++	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
++		index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
++
++		port = devlink_port_get_by_index(devlink, index);
++		if (!port) {
++			err = -ENODEV;
++			goto out_unlock;
++		}
++	}
++
++	region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
++
++	if (port)
++		region = devlink_port_region_get_by_name(port, region_name);
++	else
++		region = devlink_region_get_by_name(devlink, region_name);
++
++	if (!region) {
++		err = -EINVAL;
++		goto out_unlock;
++	}
++
++	if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
++	    attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
++		if (!start_offset)
++			start_offset =
++				nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
++
++		end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
++		end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
++	}
++
++	if (end_offset > region->size)
++		end_offset = region->size;
++
++	/* return 0 if there is no further data to read */
++	if (start_offset == end_offset) {
++		err = 0;
++		goto out_unlock;
++	}
++
++	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
++			  &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
++			  DEVLINK_CMD_REGION_READ);
++	if (!hdr) {
++		err = -EMSGSIZE;
++		goto out_unlock;
++	}
++
++	err = devlink_nl_put_handle(skb, devlink);
++	if (err)
++		goto nla_put_failure;
++
++	if (region->port) {
++		err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
++				  region->port->index);
++		if (err)
++			goto nla_put_failure;
++	}
++
++	err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
++	if (err)
++		goto nla_put_failure;
++
++	chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
++	if (!chunks_attr) {
++		err = -EMSGSIZE;
++		goto nla_put_failure;
++	}
++
++	err = devlink_nl_region_read_snapshot_fill(skb, devlink,
++						   region, attrs,
++						   start_offset,
++						   end_offset, &ret_offset);
++
++	if (err && err != -EMSGSIZE)
++		goto nla_put_failure;
++
++	/* Check if there was any progress done to prevent infinite loop */
++	if (ret_offset == start_offset) {
++		err = -EINVAL;
++		goto nla_put_failure;
++	}
++
++	*((u64 *)&cb->args[0]) = ret_offset;
++
++	nla_nest_end(skb, chunks_attr);
++	genlmsg_end(skb, hdr);
++	devl_unlock(devlink);
++	devlink_put(devlink);
++	return skb->len;
++
++nla_put_failure:
++	genlmsg_cancel(skb, hdr);
++out_unlock:
++	devl_unlock(devlink);
++	devlink_put(devlink);
++	return err;
++}
++
++int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
++{
++	if (!req->msg)
++		return 0;
++	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
++}
++EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
++
++int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
++{
++	if (!req->msg)
++		return 0;
++	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
++}
++EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
++
++int devlink_info_board_serial_number_put(struct devlink_info_req *req,
++					 const char *bsn)
++{
++	if (!req->msg)
++		return 0;
++	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
++			      bsn);
++}
++EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
++
++static int devlink_info_version_put(struct devlink_info_req *req, int attr,
++				    const char *version_name,
++				    const char *version_value,
++				    enum devlink_info_version_type version_type)
++{
++	struct nlattr *nest;
++	int err;
++
++	if (req->version_cb)
++		req->version_cb(version_name, version_type,
++				req->version_cb_priv);
++
++	if (!req->msg)
++		return 0;
++
++	nest = nla_nest_start_noflag(req->msg, attr);
++	if (!nest)
++		return -EMSGSIZE;
++
++	err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
++			     version_name);
++	if (err)
++		goto nla_put_failure;
++
++	err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
++			     version_value);
++	if (err)
++		goto nla_put_failure;
++
++	nla_nest_end(req->msg, nest);
++
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(req->msg, nest);
++	return err;
++}
++
++int devlink_info_version_fixed_put(struct devlink_info_req *req,
++				   const char *version_name,
++				   const char *version_value)
++{
++	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
++					version_name, version_value,
++					DEVLINK_INFO_VERSION_TYPE_NONE);
++}
++EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
++
++int devlink_info_version_stored_put(struct devlink_info_req *req,
++				    const char *version_name,
++				    const char *version_value)
++{
++	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
++					version_name, version_value,
++					DEVLINK_INFO_VERSION_TYPE_NONE);
++}
++EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
++
++int devlink_info_version_stored_put_ext(struct devlink_info_req *req,
++					const char *version_name,
++					const char *version_value,
++					enum devlink_info_version_type version_type)
++{
++	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
++					version_name, version_value,
++					version_type);
++}
++EXPORT_SYMBOL_GPL(devlink_info_version_stored_put_ext);
++
++int devlink_info_version_running_put(struct devlink_info_req *req,
++				     const char *version_name,
++				     const char *version_value)
++{
++	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
++					version_name, version_value,
++					DEVLINK_INFO_VERSION_TYPE_NONE);
++}
++EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
++
++int devlink_info_version_running_put_ext(struct devlink_info_req *req,
++					 const char *version_name,
++					 const char *version_value,
++					 enum devlink_info_version_type version_type)
++{
++	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
++					version_name, version_value,
++					version_type);
++}
++EXPORT_SYMBOL_GPL(devlink_info_version_running_put_ext);
++
++static int
++devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
++		     enum devlink_command cmd, u32 portid,
++		     u32 seq, int flags, struct netlink_ext_ack *extack)
++{
++	struct devlink_info_req req = {};
++	void *hdr;
++	int err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	err = -EMSGSIZE;
++	if (devlink_nl_put_handle(msg, devlink))
++		goto err_cancel_msg;
++
++	req.msg = msg;
++	err = devlink->ops->info_get(devlink, &req, extack);
++	if (err)
++		goto err_cancel_msg;
++
++	genlmsg_end(msg, hdr);
++	return 0;
++
++err_cancel_msg:
++	genlmsg_cancel(msg, hdr);
++	return err;
++}
++
++static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct sk_buff *msg;
++	int err;
++
++	if (!devlink->ops->info_get)
++		return -EOPNOTSUPP;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
++				   info->snd_portid, info->snd_seq, 0,
++				   info->extack);
++	if (err) {
++		nlmsg_free(msg);
++		return err;
++	}
++
++	return genlmsg_reply(msg, info);
++}
++
++static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
++					  struct netlink_callback *cb)
++{
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err = 0;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		if (idx < start || !devlink->ops->info_get)
++			goto inc;
++
++		devl_lock(devlink);
++		err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
++					   NETLINK_CB(cb->skb).portid,
++					   cb->nlh->nlmsg_seq, NLM_F_MULTI,
++					   cb->extack);
++		devl_unlock(devlink);
++		if (err == -EOPNOTSUPP)
++			err = 0;
++		else if (err) {
++			devlink_put(devlink);
++			break;
++		}
++inc:
++		idx++;
++		devlink_put(devlink);
++	}
++
++	if (err != -EMSGSIZE)
++		return err;
++
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++struct devlink_fmsg_item {
++	struct list_head list;
++	int attrtype;
++	u8 nla_type;
++	u16 len;
++	int value[];
++};
++
++struct devlink_fmsg {
++	struct list_head item_list;
++	bool putting_binary; /* This flag forces enclosing of binary data
++			      * in an array brackets. It forces using
++			      * of designated API:
++			      * devlink_fmsg_binary_pair_nest_start()
++			      * devlink_fmsg_binary_pair_nest_end()
++			      */
++};
++
++static struct devlink_fmsg *devlink_fmsg_alloc(void)
++{
++	struct devlink_fmsg *fmsg;
++
++	fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
++	if (!fmsg)
++		return NULL;
++
++	INIT_LIST_HEAD(&fmsg->item_list);
++
++	return fmsg;
++}
++
++static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
++{
++	struct devlink_fmsg_item *item, *tmp;
++
++	list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
++		list_del(&item->list);
++		kfree(item);
++	}
++	kfree(fmsg);
++}
++
++static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
++				    int attrtype)
++{
++	struct devlink_fmsg_item *item;
++
++	item = kzalloc(sizeof(*item), GFP_KERNEL);
++	if (!item)
++		return -ENOMEM;
++
++	item->attrtype = attrtype;
++	list_add_tail(&item->list, &fmsg->item_list);
++
++	return 0;
++}
++
++int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
++
++static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
++}
++
++int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_nest_end(fmsg);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
++
++#define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
++
++static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
++{
++	struct devlink_fmsg_item *item;
++
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
++		return -EMSGSIZE;
++
++	item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
++	if (!item)
++		return -ENOMEM;
++
++	item->nla_type = NLA_NUL_STRING;
++	item->len = strlen(name) + 1;
++	item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
++	memcpy(&item->value, name, item->len);
++	list_add_tail(&item->list, &fmsg->item_list);
++
++	return 0;
++}
++
++int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
++{
++	int err;
++
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_put_name(fmsg, name);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
++
++int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_nest_end(fmsg);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
++
++int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
++				     const char *name)
++{
++	int err;
++
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	err = devlink_fmsg_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
++
++int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
++{
++	int err;
++
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	err = devlink_fmsg_nest_end(fmsg);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_nest_end(fmsg);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
++
++int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
++					const char *name)
++{
++	int err;
++
++	err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	fmsg->putting_binary = true;
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
++
++int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
++{
++	if (!fmsg->putting_binary)
++		return -EINVAL;
++
++	fmsg->putting_binary = false;
++	return devlink_fmsg_arr_pair_nest_end(fmsg);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
++
++static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
++				  const void *value, u16 value_len,
++				  u8 value_nla_type)
++{
++	struct devlink_fmsg_item *item;
++
++	if (value_len > DEVLINK_FMSG_MAX_SIZE)
++		return -EMSGSIZE;
++
++	item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
++	if (!item)
++		return -ENOMEM;
++
++	item->nla_type = value_nla_type;
++	item->len = value_len;
++	item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
++	memcpy(&item->value, value, item->len);
++	list_add_tail(&item->list, &fmsg->item_list);
++
++	return 0;
++}
++
++static int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
++}
++
++static int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
++}
++
++int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
++
++static int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
++}
++
++int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
++{
++	if (fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
++				      NLA_NUL_STRING);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
++
++int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
++			    u16 value_len)
++{
++	if (!fmsg->putting_binary)
++		return -EINVAL;
++
++	return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
++
++int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
++			       bool value)
++{
++	int err;
++
++	err = devlink_fmsg_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_bool_put(fmsg, value);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_pair_nest_end(fmsg);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
++
++int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
++			     u8 value)
++{
++	int err;
++
++	err = devlink_fmsg_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_u8_put(fmsg, value);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_pair_nest_end(fmsg);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
++
++int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
++			      u32 value)
++{
++	int err;
++
++	err = devlink_fmsg_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_u32_put(fmsg, value);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_pair_nest_end(fmsg);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
++
++int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
++			      u64 value)
++{
++	int err;
++
++	err = devlink_fmsg_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_u64_put(fmsg, value);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_pair_nest_end(fmsg);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
++
++int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
++				 const char *value)
++{
++	int err;
++
++	err = devlink_fmsg_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_string_put(fmsg, value);
++	if (err)
++		return err;
++
++	err = devlink_fmsg_pair_nest_end(fmsg);
++	if (err)
++		return err;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
++
++int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
++				 const void *value, u32 value_len)
++{
++	u32 data_size;
++	int end_err;
++	u32 offset;
++	int err;
++
++	err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
++	if (err)
++		return err;
++
++	for (offset = 0; offset < value_len; offset += data_size) {
++		data_size = value_len - offset;
++		if (data_size > DEVLINK_FMSG_MAX_SIZE)
++			data_size = DEVLINK_FMSG_MAX_SIZE;
++		err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
++		if (err)
++			break;
++		/* Exit from loop with a break (instead of
++		 * return) to make sure putting_binary is turned off in
++		 * devlink_fmsg_binary_pair_nest_end
++		 */
++	}
++
++	end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
++	if (end_err)
++		err = end_err;
++
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
++
++static int
++devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
++{
++	switch (msg->nla_type) {
++	case NLA_FLAG:
++	case NLA_U8:
++	case NLA_U32:
++	case NLA_U64:
++	case NLA_NUL_STRING:
++	case NLA_BINARY:
++		return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
++				  msg->nla_type);
++	default:
++		return -EINVAL;
++	}
++}
++
++static int
++devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
++{
++	int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
++	u8 tmp;
++
++	switch (msg->nla_type) {
++	case NLA_FLAG:
++		/* Always provide flag data, regardless of its value */
++		tmp = *(bool *) msg->value;
++
++		return nla_put_u8(skb, attrtype, tmp);
++	case NLA_U8:
++		return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
++	case NLA_U32:
++		return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
++	case NLA_U64:
++		return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
++					 DEVLINK_ATTR_PAD);
++	case NLA_NUL_STRING:
++		return nla_put_string(skb, attrtype, (char *) &msg->value);
++	case NLA_BINARY:
++		return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
++	default:
++		return -EINVAL;
++	}
++}
++
++static int
++devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
++			 int *start)
++{
++	struct devlink_fmsg_item *item;
++	struct nlattr *fmsg_nlattr;
++	int i = 0;
++	int err;
++
++	fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
++	if (!fmsg_nlattr)
++		return -EMSGSIZE;
++
++	list_for_each_entry(item, &fmsg->item_list, list) {
++		if (i < *start) {
++			i++;
++			continue;
++		}
++
++		switch (item->attrtype) {
++		case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
++		case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
++		case DEVLINK_ATTR_FMSG_ARR_NEST_START:
++		case DEVLINK_ATTR_FMSG_NEST_END:
++			err = nla_put_flag(skb, item->attrtype);
++			break;
++		case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
++			err = devlink_fmsg_item_fill_type(item, skb);
++			if (err)
++				break;
++			err = devlink_fmsg_item_fill_data(item, skb);
++			break;
++		case DEVLINK_ATTR_FMSG_OBJ_NAME:
++			err = nla_put_string(skb, item->attrtype,
++					     (char *) &item->value);
++			break;
++		default:
++			err = -EINVAL;
++			break;
++		}
++		if (!err)
++			*start = ++i;
++		else
++			break;
++	}
++
++	nla_nest_end(skb, fmsg_nlattr);
++	return err;
++}
++
++static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
++			    struct genl_info *info,
++			    enum devlink_command cmd, int flags)
++{
++	struct nlmsghdr *nlh;
++	struct sk_buff *skb;
++	bool last = false;
++	int index = 0;
++	void *hdr;
++	int err;
++
++	while (!last) {
++		int tmp_index = index;
++
++		skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
++		if (!skb)
++			return -ENOMEM;
++
++		hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
++				  &devlink_nl_family, flags | NLM_F_MULTI, cmd);
++		if (!hdr) {
++			err = -EMSGSIZE;
++			goto nla_put_failure;
++		}
++
++		err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
++		if (!err)
++			last = true;
++		else if (err != -EMSGSIZE || tmp_index == index)
++			goto nla_put_failure;
++
++		genlmsg_end(skb, hdr);
++		err = genlmsg_reply(skb, info);
++		if (err)
++			return err;
++	}
++
++	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!skb)
++		return -ENOMEM;
++	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
++			NLMSG_DONE, 0, flags | NLM_F_MULTI);
++	if (!nlh) {
++		err = -EMSGSIZE;
++		goto nla_put_failure;
++	}
++
++	return genlmsg_reply(skb, info);
++
++nla_put_failure:
++	nlmsg_free(skb);
++	return err;
++}
++
++static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
++			       struct netlink_callback *cb,
++			       enum devlink_command cmd)
++{
++	int index = cb->args[0];
++	int tmp_index = index;
++	void *hdr;
++	int err;
++
++	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
++			  &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
++	if (!hdr) {
++		err = -EMSGSIZE;
++		goto nla_put_failure;
++	}
++
++	err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
++	if ((err && err != -EMSGSIZE) || tmp_index == index)
++		goto nla_put_failure;
++
++	cb->args[0] = index;
++	genlmsg_end(skb, hdr);
++	return skb->len;
++
++nla_put_failure:
++	genlmsg_cancel(skb, hdr);
++	return err;
++}
++
++struct devlink_health_reporter {
++	struct list_head list;
++	void *priv;
++	const struct devlink_health_reporter_ops *ops;
++	struct devlink *devlink;
++	struct devlink_port *devlink_port;
++	struct devlink_fmsg *dump_fmsg;
++	struct mutex dump_lock; /* lock parallel read/write from dump buffers */
++	u64 graceful_period;
++	bool auto_recover;
++	bool auto_dump;
++	u8 health_state;
++	u64 dump_ts;
++	u64 dump_real_ts;
++	u64 error_count;
++	u64 recovery_count;
++	u64 last_recovery_ts;
++	refcount_t refcount;
++};
++
++void *
++devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
++{
++	return reporter->priv;
++}
++EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
++
++static struct devlink_health_reporter *
++__devlink_health_reporter_find_by_name(struct list_head *reporter_list,
++				       struct mutex *list_lock,
++				       const char *reporter_name)
++{
++	struct devlink_health_reporter *reporter;
++
++	lockdep_assert_held(list_lock);
++	list_for_each_entry(reporter, reporter_list, list)
++		if (!strcmp(reporter->ops->name, reporter_name))
++			return reporter;
++	return NULL;
++}
++
++static struct devlink_health_reporter *
++devlink_health_reporter_find_by_name(struct devlink *devlink,
++				     const char *reporter_name)
++{
++	return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
++						      &devlink->reporters_lock,
++						      reporter_name);
++}
++
++static struct devlink_health_reporter *
++devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
++					  const char *reporter_name)
++{
++	return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
++						      &devlink_port->reporters_lock,
++						      reporter_name);
++}
++
++static struct devlink_health_reporter *
++__devlink_health_reporter_create(struct devlink *devlink,
++				 const struct devlink_health_reporter_ops *ops,
++				 u64 graceful_period, void *priv)
++{
++	struct devlink_health_reporter *reporter;
++
++	if (WARN_ON(graceful_period && !ops->recover))
++		return ERR_PTR(-EINVAL);
++
++	reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
++	if (!reporter)
++		return ERR_PTR(-ENOMEM);
++
++	reporter->priv = priv;
++	reporter->ops = ops;
++	reporter->devlink = devlink;
++	reporter->graceful_period = graceful_period;
++	reporter->auto_recover = !!ops->recover;
++	reporter->auto_dump = !!ops->dump;
++	mutex_init(&reporter->dump_lock);
++	refcount_set(&reporter->refcount, 1);
++	return reporter;
++}
++
++/**
++ *	devlink_port_health_reporter_create - create devlink health reporter for
++ *	                                      specified port instance
++ *
++ *	@port: devlink_port which should contain the new reporter
++ *	@ops: ops
++ *	@graceful_period: to avoid recovery loops, in msecs
++ *	@priv: priv
++ */
++struct devlink_health_reporter *
++devlink_port_health_reporter_create(struct devlink_port *port,
++				    const struct devlink_health_reporter_ops *ops,
++				    u64 graceful_period, void *priv)
++{
++	struct devlink_health_reporter *reporter;
++
++	mutex_lock(&port->reporters_lock);
++	if (__devlink_health_reporter_find_by_name(&port->reporter_list,
++						   &port->reporters_lock, ops->name)) {
++		reporter = ERR_PTR(-EEXIST);
++		goto unlock;
++	}
++
++	reporter = __devlink_health_reporter_create(port->devlink, ops,
++						    graceful_period, priv);
++	if (IS_ERR(reporter))
++		goto unlock;
++
++	reporter->devlink_port = port;
++	list_add_tail(&reporter->list, &port->reporter_list);
++unlock:
++	mutex_unlock(&port->reporters_lock);
++	return reporter;
++}
++EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
++
++/**
++ *	devlink_health_reporter_create - create devlink health reporter
++ *
++ *	@devlink: devlink
++ *	@ops: ops
++ *	@graceful_period: to avoid recovery loops, in msecs
++ *	@priv: priv
++ */
++struct devlink_health_reporter *
++devlink_health_reporter_create(struct devlink *devlink,
++			       const struct devlink_health_reporter_ops *ops,
++			       u64 graceful_period, void *priv)
++{
++	struct devlink_health_reporter *reporter;
++
++	mutex_lock(&devlink->reporters_lock);
++	if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
++		reporter = ERR_PTR(-EEXIST);
++		goto unlock;
++	}
++
++	reporter = __devlink_health_reporter_create(devlink, ops,
++						    graceful_period, priv);
++	if (IS_ERR(reporter))
++		goto unlock;
++
++	list_add_tail(&reporter->list, &devlink->reporter_list);
++unlock:
++	mutex_unlock(&devlink->reporters_lock);
++	return reporter;
++}
++EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
++
++static void
++devlink_health_reporter_free(struct devlink_health_reporter *reporter)
++{
++	mutex_destroy(&reporter->dump_lock);
++	if (reporter->dump_fmsg)
++		devlink_fmsg_free(reporter->dump_fmsg);
++	kfree(reporter);
++}
++
++static void
++devlink_health_reporter_put(struct devlink_health_reporter *reporter)
++{
++	if (refcount_dec_and_test(&reporter->refcount))
++		devlink_health_reporter_free(reporter);
++}
++
++static void
++__devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
++{
++	list_del(&reporter->list);
++	devlink_health_reporter_put(reporter);
++}
++
++/**
++ *	devlink_health_reporter_destroy - destroy devlink health reporter
++ *
++ *	@reporter: devlink health reporter to destroy
++ */
++void
++devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
++{
++	struct mutex *lock = &reporter->devlink->reporters_lock;
++
++	mutex_lock(lock);
++	__devlink_health_reporter_destroy(reporter);
++	mutex_unlock(lock);
++}
++EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
++
++/**
++ *	devlink_port_health_reporter_destroy - destroy devlink port health reporter
++ *
++ *	@reporter: devlink health reporter to destroy
++ */
++void
++devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
++{
++	struct mutex *lock = &reporter->devlink_port->reporters_lock;
++
++	mutex_lock(lock);
++	__devlink_health_reporter_destroy(reporter);
++	mutex_unlock(lock);
++}
++EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
++
++static int
++devlink_nl_health_reporter_fill(struct sk_buff *msg,
++				struct devlink_health_reporter *reporter,
++				enum devlink_command cmd, u32 portid,
++				u32 seq, int flags)
++{
++	struct devlink *devlink = reporter->devlink;
++	struct nlattr *reporter_attr;
++	void *hdr;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto genlmsg_cancel;
++
++	if (reporter->devlink_port) {
++		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
++			goto genlmsg_cancel;
++	}
++	reporter_attr = nla_nest_start_noflag(msg,
++					      DEVLINK_ATTR_HEALTH_REPORTER);
++	if (!reporter_attr)
++		goto genlmsg_cancel;
++	if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
++			   reporter->ops->name))
++		goto reporter_nest_cancel;
++	if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
++		       reporter->health_state))
++		goto reporter_nest_cancel;
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
++			      reporter->error_count, DEVLINK_ATTR_PAD))
++		goto reporter_nest_cancel;
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
++			      reporter->recovery_count, DEVLINK_ATTR_PAD))
++		goto reporter_nest_cancel;
++	if (reporter->ops->recover &&
++	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
++			      reporter->graceful_period,
++			      DEVLINK_ATTR_PAD))
++		goto reporter_nest_cancel;
++	if (reporter->ops->recover &&
++	    nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
++		       reporter->auto_recover))
++		goto reporter_nest_cancel;
++	if (reporter->dump_fmsg &&
++	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
++			      jiffies_to_msecs(reporter->dump_ts),
++			      DEVLINK_ATTR_PAD))
++		goto reporter_nest_cancel;
++	if (reporter->dump_fmsg &&
++	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
++			      reporter->dump_real_ts, DEVLINK_ATTR_PAD))
++		goto reporter_nest_cancel;
++	if (reporter->ops->dump &&
++	    nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
++		       reporter->auto_dump))
++		goto reporter_nest_cancel;
++
++	nla_nest_end(msg, reporter_attr);
++	genlmsg_end(msg, hdr);
++	return 0;
++
++reporter_nest_cancel:
++	nla_nest_end(msg, reporter_attr);
++genlmsg_cancel:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static void devlink_recover_notify(struct devlink_health_reporter *reporter,
++				   enum devlink_command cmd)
++{
++	struct devlink *devlink = reporter->devlink;
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
++	WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
++				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++void
++devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
++{
++	reporter->recovery_count++;
++	reporter->last_recovery_ts = jiffies;
++}
++EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
++
++static int
++devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
++				void *priv_ctx, struct netlink_ext_ack *extack)
++{
++	int err;
++
++	if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
++		return 0;
++
++	if (!reporter->ops->recover)
++		return -EOPNOTSUPP;
++
++	err = reporter->ops->recover(reporter, priv_ctx, extack);
++	if (err)
++		return err;
++
++	devlink_health_reporter_recovery_done(reporter);
++	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
++	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
++
++	return 0;
++}
++
++static void
++devlink_health_dump_clear(struct devlink_health_reporter *reporter)
++{
++	if (!reporter->dump_fmsg)
++		return;
++	devlink_fmsg_free(reporter->dump_fmsg);
++	reporter->dump_fmsg = NULL;
++}
++
++static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
++				  void *priv_ctx,
++				  struct netlink_ext_ack *extack)
++{
++	int err;
++
++	if (!reporter->ops->dump)
++		return 0;
++
++	if (reporter->dump_fmsg)
++		return 0;
++
++	reporter->dump_fmsg = devlink_fmsg_alloc();
++	if (!reporter->dump_fmsg) {
++		err = -ENOMEM;
++		return err;
++	}
++
++	err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
++	if (err)
++		goto dump_err;
++
++	err = reporter->ops->dump(reporter, reporter->dump_fmsg,
++				  priv_ctx, extack);
++	if (err)
++		goto dump_err;
++
++	err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
++	if (err)
++		goto dump_err;
++
++	reporter->dump_ts = jiffies;
++	reporter->dump_real_ts = ktime_get_real_ns();
++
++	return 0;
++
++dump_err:
++	devlink_health_dump_clear(reporter);
++	return err;
++}
++
++int devlink_health_report(struct devlink_health_reporter *reporter,
++			  const char *msg, void *priv_ctx)
++{
++	enum devlink_health_reporter_state prev_health_state;
++	struct devlink *devlink = reporter->devlink;
++	unsigned long recover_ts_threshold;
++	int ret;
++
++	/* write a log message of the current error */
++	WARN_ON(!msg);
++	trace_devlink_health_report(devlink, reporter->ops->name, msg);
++	reporter->error_count++;
++	prev_health_state = reporter->health_state;
++	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
++	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
++
++	/* abort if the previous error wasn't recovered */
++	recover_ts_threshold = reporter->last_recovery_ts +
++			       msecs_to_jiffies(reporter->graceful_period);
++	if (reporter->auto_recover &&
++	    (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
++	     (reporter->last_recovery_ts && reporter->recovery_count &&
++	      time_is_after_jiffies(recover_ts_threshold)))) {
++		trace_devlink_health_recover_aborted(devlink,
++						     reporter->ops->name,
++						     reporter->health_state,
++						     jiffies -
++						     reporter->last_recovery_ts);
++		return -ECANCELED;
++	}
++
++	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
++
++	if (reporter->auto_dump) {
++		mutex_lock(&reporter->dump_lock);
++		/* store current dump of current error, for later analysis */
++		devlink_health_do_dump(reporter, priv_ctx, NULL);
++		mutex_unlock(&reporter->dump_lock);
++	}
++
++	if (!reporter->auto_recover)
++		return 0;
++
++	devl_lock(devlink);
++	ret = devlink_health_reporter_recover(reporter, priv_ctx, NULL);
++	devl_unlock(devlink);
++
++	return ret;
++}
++EXPORT_SYMBOL_GPL(devlink_health_report);
++
++static struct devlink_health_reporter *
++devlink_health_reporter_get_from_attrs(struct devlink *devlink,
++				       struct nlattr **attrs)
++{
++	struct devlink_health_reporter *reporter;
++	struct devlink_port *devlink_port;
++	char *reporter_name;
++
++	if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
++		return NULL;
++
++	reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
++	devlink_port = devlink_port_get_from_attrs(devlink, attrs);
++	if (IS_ERR(devlink_port)) {
++		mutex_lock(&devlink->reporters_lock);
++		reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
++		if (reporter)
++			refcount_inc(&reporter->refcount);
++		mutex_unlock(&devlink->reporters_lock);
++	} else {
++		mutex_lock(&devlink_port->reporters_lock);
++		reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
++		if (reporter)
++			refcount_inc(&reporter->refcount);
++		mutex_unlock(&devlink_port->reporters_lock);
++	}
++
++	return reporter;
++}
++
++static struct devlink_health_reporter *
++devlink_health_reporter_get_from_info(struct devlink *devlink,
++				      struct genl_info *info)
++{
++	return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
++}
++
++static struct devlink_health_reporter *
++devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
++{
++	const struct genl_dumpit_info *info = genl_dumpit_info(cb);
++	struct devlink_health_reporter *reporter;
++	struct nlattr **attrs = info->attrs;
++	struct devlink *devlink;
++
++	devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
++	if (IS_ERR(devlink))
++		return NULL;
++
++	reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
++	devlink_put(devlink);
++	return reporter;
++}
++
++void
++devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
++				     enum devlink_health_reporter_state state)
++{
++	if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
++		    state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
++		return;
++
++	if (reporter->health_state == state)
++		return;
++
++	reporter->health_state = state;
++	trace_devlink_health_reporter_state_update(reporter->devlink,
++						   reporter->ops->name, state);
++	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
++}
++EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
++
++static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
++						   struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_health_reporter *reporter;
++	struct sk_buff *msg;
++	int err;
++
++	reporter = devlink_health_reporter_get_from_info(devlink, info);
++	if (!reporter)
++		return -EINVAL;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg) {
++		err = -ENOMEM;
++		goto out;
++	}
++
++	err = devlink_nl_health_reporter_fill(msg, reporter,
++					      DEVLINK_CMD_HEALTH_REPORTER_GET,
++					      info->snd_portid, info->snd_seq,
++					      0);
++	if (err) {
++		nlmsg_free(msg);
++		goto out;
++	}
++
++	err = genlmsg_reply(msg, info);
++out:
++	devlink_health_reporter_put(reporter);
++	return err;
++}
++
++static int
++devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
++					  struct netlink_callback *cb)
++{
++	struct devlink_health_reporter *reporter;
++	struct devlink_port *port;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		mutex_lock(&devlink->reporters_lock);
++		list_for_each_entry(reporter, &devlink->reporter_list,
++				    list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_health_reporter_fill(
++				msg, reporter, DEVLINK_CMD_HEALTH_REPORTER_GET,
++				NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
++				NLM_F_MULTI);
++			if (err) {
++				mutex_unlock(&devlink->reporters_lock);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		mutex_unlock(&devlink->reporters_lock);
++		devlink_put(devlink);
++	}
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(port, &devlink->port_list, list) {
++			mutex_lock(&port->reporters_lock);
++			list_for_each_entry(reporter, &port->reporter_list, list) {
++				if (idx < start) {
++					idx++;
++					continue;
++				}
++				err = devlink_nl_health_reporter_fill(
++					msg, reporter,
++					DEVLINK_CMD_HEALTH_REPORTER_GET,
++					NETLINK_CB(cb->skb).portid,
++					cb->nlh->nlmsg_seq, NLM_F_MULTI);
++				if (err) {
++					mutex_unlock(&port->reporters_lock);
++					devl_unlock(devlink);
++					devlink_put(devlink);
++					goto out;
++				}
++				idx++;
++			}
++			mutex_unlock(&port->reporters_lock);
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int
++devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_health_reporter *reporter;
++	int err;
++
++	reporter = devlink_health_reporter_get_from_info(devlink, info);
++	if (!reporter)
++		return -EINVAL;
++
++	if (!reporter->ops->recover &&
++	    (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
++	     info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
++		err = -EOPNOTSUPP;
++		goto out;
++	}
++	if (!reporter->ops->dump &&
++	    info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
++		err = -EOPNOTSUPP;
++		goto out;
++	}
++
++	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
++		reporter->graceful_period =
++			nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
++
++	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
++		reporter->auto_recover =
++			nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
++
++	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
++		reporter->auto_dump =
++		nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
++
++	devlink_health_reporter_put(reporter);
++	return 0;
++out:
++	devlink_health_reporter_put(reporter);
++	return err;
++}
++
++static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
++						       struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_health_reporter *reporter;
++	int err;
++
++	reporter = devlink_health_reporter_get_from_info(devlink, info);
++	if (!reporter)
++		return -EINVAL;
++
++	err = devlink_health_reporter_recover(reporter, NULL, info->extack);
++
++	devlink_health_reporter_put(reporter);
++	return err;
++}
++
++static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
++							struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_health_reporter *reporter;
++	struct devlink_fmsg *fmsg;
++	int err;
++
++	reporter = devlink_health_reporter_get_from_info(devlink, info);
++	if (!reporter)
++		return -EINVAL;
++
++	if (!reporter->ops->diagnose) {
++		devlink_health_reporter_put(reporter);
++		return -EOPNOTSUPP;
++	}
++
++	fmsg = devlink_fmsg_alloc();
++	if (!fmsg) {
++		devlink_health_reporter_put(reporter);
++		return -ENOMEM;
++	}
++
++	err = devlink_fmsg_obj_nest_start(fmsg);
++	if (err)
++		goto out;
++
++	err = reporter->ops->diagnose(reporter, fmsg, info->extack);
++	if (err)
++		goto out;
++
++	err = devlink_fmsg_obj_nest_end(fmsg);
++	if (err)
++		goto out;
++
++	err = devlink_fmsg_snd(fmsg, info,
++			       DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
++
++out:
++	devlink_fmsg_free(fmsg);
++	devlink_health_reporter_put(reporter);
++	return err;
++}
++
++static int
++devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
++					       struct netlink_callback *cb)
++{
++	struct devlink_health_reporter *reporter;
++	u64 start = cb->args[0];
++	int err;
++
++	reporter = devlink_health_reporter_get_from_cb(cb);
++	if (!reporter)
++		return -EINVAL;
++
++	if (!reporter->ops->dump) {
++		err = -EOPNOTSUPP;
++		goto out;
++	}
++	mutex_lock(&reporter->dump_lock);
++	if (!start) {
++		err = devlink_health_do_dump(reporter, NULL, cb->extack);
++		if (err)
++			goto unlock;
++		cb->args[1] = reporter->dump_ts;
++	}
++	if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
++		NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
++		err = -EAGAIN;
++		goto unlock;
++	}
++
++	err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
++				  DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
++unlock:
++	mutex_unlock(&reporter->dump_lock);
++out:
++	devlink_health_reporter_put(reporter);
++	return err;
++}
++
++static int
++devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
++					       struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_health_reporter *reporter;
++
++	reporter = devlink_health_reporter_get_from_info(devlink, info);
++	if (!reporter)
++		return -EINVAL;
++
++	if (!reporter->ops->dump) {
++		devlink_health_reporter_put(reporter);
++		return -EOPNOTSUPP;
++	}
++
++	mutex_lock(&reporter->dump_lock);
++	devlink_health_dump_clear(reporter);
++	mutex_unlock(&reporter->dump_lock);
++	devlink_health_reporter_put(reporter);
++	return 0;
++}
++
++static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
++						    struct genl_info *info)
++{
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_health_reporter *reporter;
++	int err;
++
++	reporter = devlink_health_reporter_get_from_info(devlink, info);
++	if (!reporter)
++		return -EINVAL;
++
++	if (!reporter->ops->test) {
++		devlink_health_reporter_put(reporter);
++		return -EOPNOTSUPP;
++	}
++
++	err = reporter->ops->test(reporter, info->extack);
++
++	devlink_health_reporter_put(reporter);
++	return err;
++}
++
++struct devlink_stats {
++	u64_stats_t rx_bytes;
++	u64_stats_t rx_packets;
++	struct u64_stats_sync syncp;
++};
++
++/**
++ * struct devlink_trap_policer_item - Packet trap policer attributes.
++ * @policer: Immutable packet trap policer attributes.
++ * @rate: Rate in packets / sec.
++ * @burst: Burst size in packets.
++ * @list: trap_policer_list member.
++ *
++ * Describes packet trap policer attributes. Created by devlink during trap
++ * policer registration.
++ */
++struct devlink_trap_policer_item {
++	const struct devlink_trap_policer *policer;
++	u64 rate;
++	u64 burst;
++	struct list_head list;
++};
++
++/**
++ * struct devlink_trap_group_item - Packet trap group attributes.
++ * @group: Immutable packet trap group attributes.
++ * @policer_item: Associated policer item. Can be NULL.
++ * @list: trap_group_list member.
++ * @stats: Trap group statistics.
++ *
++ * Describes packet trap group attributes. Created by devlink during trap
++ * group registration.
++ */
++struct devlink_trap_group_item {
++	const struct devlink_trap_group *group;
++	struct devlink_trap_policer_item *policer_item;
++	struct list_head list;
++	struct devlink_stats __percpu *stats;
++};
++
++/**
++ * struct devlink_trap_item - Packet trap attributes.
++ * @trap: Immutable packet trap attributes.
++ * @group_item: Associated group item.
++ * @list: trap_list member.
++ * @action: Trap action.
++ * @stats: Trap statistics.
++ * @priv: Driver private information.
++ *
++ * Describes both mutable and immutable packet trap attributes. Created by
++ * devlink during trap registration and used for all trap related operations.
++ */
++struct devlink_trap_item {
++	const struct devlink_trap *trap;
++	struct devlink_trap_group_item *group_item;
++	struct list_head list;
++	enum devlink_trap_action action;
++	struct devlink_stats __percpu *stats;
++	void *priv;
++};
++
++static struct devlink_trap_policer_item *
++devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
++{
++	struct devlink_trap_policer_item *policer_item;
++
++	list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
++		if (policer_item->policer->id == id)
++			return policer_item;
++	}
++
++	return NULL;
++}
++
++static struct devlink_trap_item *
++devlink_trap_item_lookup(struct devlink *devlink, const char *name)
++{
++	struct devlink_trap_item *trap_item;
++
++	list_for_each_entry(trap_item, &devlink->trap_list, list) {
++		if (!strcmp(trap_item->trap->name, name))
++			return trap_item;
++	}
++
++	return NULL;
++}
++
++static struct devlink_trap_item *
++devlink_trap_item_get_from_info(struct devlink *devlink,
++				struct genl_info *info)
++{
++	struct nlattr *attr;
++
++	if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
++		return NULL;
++	attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
++
++	return devlink_trap_item_lookup(devlink, nla_data(attr));
++}
++
++static int
++devlink_trap_action_get_from_info(struct genl_info *info,
++				  enum devlink_trap_action *p_trap_action)
++{
++	u8 val;
++
++	val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
++	switch (val) {
++	case DEVLINK_TRAP_ACTION_DROP:
++	case DEVLINK_TRAP_ACTION_TRAP:
++	case DEVLINK_TRAP_ACTION_MIRROR:
++		*p_trap_action = val;
++		break;
++	default:
++		return -EINVAL;
++	}
++
++	return 0;
++}
++
++static int devlink_trap_metadata_put(struct sk_buff *msg,
++				     const struct devlink_trap *trap)
++{
++	struct nlattr *attr;
++
++	attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
++	if (!attr)
++		return -EMSGSIZE;
++
++	if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
++	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
++		goto nla_put_failure;
++	if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
++	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
++		goto nla_put_failure;
++
++	nla_nest_end(msg, attr);
++
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, attr);
++	return -EMSGSIZE;
++}
++
++static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
++				    struct devlink_stats *stats)
++{
++	int i;
++
++	memset(stats, 0, sizeof(*stats));
++	for_each_possible_cpu(i) {
++		struct devlink_stats *cpu_stats;
++		u64 rx_packets, rx_bytes;
++		unsigned int start;
++
++		cpu_stats = per_cpu_ptr(trap_stats, i);
++		do {
++			start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
++			rx_packets = u64_stats_read(&cpu_stats->rx_packets);
++			rx_bytes = u64_stats_read(&cpu_stats->rx_bytes);
++		} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
++
++		u64_stats_add(&stats->rx_packets, rx_packets);
++		u64_stats_add(&stats->rx_bytes, rx_bytes);
++	}
++}
++
++static int
++devlink_trap_group_stats_put(struct sk_buff *msg,
++			     struct devlink_stats __percpu *trap_stats)
++{
++	struct devlink_stats stats;
++	struct nlattr *attr;
++
++	devlink_trap_stats_read(trap_stats, &stats);
++
++	attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
++	if (!attr)
++		return -EMSGSIZE;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
++			      u64_stats_read(&stats.rx_packets),
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
++			      u64_stats_read(&stats.rx_bytes),
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	nla_nest_end(msg, attr);
++
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink,
++				  const struct devlink_trap_item *trap_item)
++{
++	struct devlink_stats stats;
++	struct nlattr *attr;
++	u64 drops = 0;
++	int err;
++
++	if (devlink->ops->trap_drop_counter_get) {
++		err = devlink->ops->trap_drop_counter_get(devlink,
++							  trap_item->trap,
++							  &drops);
++		if (err)
++			return err;
++	}
++
++	devlink_trap_stats_read(trap_item->stats, &stats);
++
++	attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
++	if (!attr)
++		return -EMSGSIZE;
++
++	if (devlink->ops->trap_drop_counter_get &&
++	    nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
++			      u64_stats_read(&stats.rx_packets),
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
++			      u64_stats_read(&stats.rx_bytes),
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	nla_nest_end(msg, attr);
++
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, attr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
++				const struct devlink_trap_item *trap_item,
++				enum devlink_command cmd, u32 portid, u32 seq,
++				int flags)
++{
++	struct devlink_trap_group_item *group_item = trap_item->group_item;
++	void *hdr;
++	int err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
++			   group_item->group->name))
++		goto nla_put_failure;
++
++	if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
++		goto nla_put_failure;
++
++	if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
++		goto nla_put_failure;
++
++	if (trap_item->trap->generic &&
++	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
++		goto nla_put_failure;
++
++	if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
++		goto nla_put_failure;
++
++	err = devlink_trap_metadata_put(msg, trap_item->trap);
++	if (err)
++		goto nla_put_failure;
++
++	err = devlink_trap_stats_put(msg, devlink, trap_item);
++	if (err)
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_trap_item *trap_item;
++	struct sk_buff *msg;
++	int err;
++
++	if (list_empty(&devlink->trap_list))
++		return -EOPNOTSUPP;
++
++	trap_item = devlink_trap_item_get_from_info(devlink, info);
++	if (!trap_item) {
++		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
++		return -ENOENT;
++	}
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_trap_fill(msg, devlink, trap_item,
++				   DEVLINK_CMD_TRAP_NEW, info->snd_portid,
++				   info->snd_seq, 0);
++	if (err)
++		goto err_trap_fill;
++
++	return genlmsg_reply(msg, info);
++
++err_trap_fill:
++	nlmsg_free(msg);
++	return err;
++}
++
++static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
++					  struct netlink_callback *cb)
++{
++	struct devlink_trap_item *trap_item;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(trap_item, &devlink->trap_list, list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_trap_fill(msg, devlink, trap_item,
++						   DEVLINK_CMD_TRAP_NEW,
++						   NETLINK_CB(cb->skb).portid,
++						   cb->nlh->nlmsg_seq,
++						   NLM_F_MULTI);
++			if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int __devlink_trap_action_set(struct devlink *devlink,
++				     struct devlink_trap_item *trap_item,
++				     enum devlink_trap_action trap_action,
++				     struct netlink_ext_ack *extack)
++{
++	int err;
++
++	if (trap_item->action != trap_action &&
++	    trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
++		NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
++		return 0;
++	}
++
++	err = devlink->ops->trap_action_set(devlink, trap_item->trap,
++					    trap_action, extack);
++	if (err)
++		return err;
++
++	trap_item->action = trap_action;
++
++	return 0;
++}
++
++static int devlink_trap_action_set(struct devlink *devlink,
++				   struct devlink_trap_item *trap_item,
++				   struct genl_info *info)
++{
++	enum devlink_trap_action trap_action;
++	int err;
++
++	if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
++		return 0;
++
++	err = devlink_trap_action_get_from_info(info, &trap_action);
++	if (err) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
++		return -EINVAL;
++	}
++
++	return __devlink_trap_action_set(devlink, trap_item, trap_action,
++					 info->extack);
++}
++
++static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
++					struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_trap_item *trap_item;
++
++	if (list_empty(&devlink->trap_list))
++		return -EOPNOTSUPP;
++
++	trap_item = devlink_trap_item_get_from_info(devlink, info);
++	if (!trap_item) {
++		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
++		return -ENOENT;
++	}
++
++	return devlink_trap_action_set(devlink, trap_item, info);
++}
++
++static struct devlink_trap_group_item *
++devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
++{
++	struct devlink_trap_group_item *group_item;
++
++	list_for_each_entry(group_item, &devlink->trap_group_list, list) {
++		if (!strcmp(group_item->group->name, name))
++			return group_item;
++	}
++
++	return NULL;
++}
++
++static struct devlink_trap_group_item *
++devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
++{
++	struct devlink_trap_group_item *group_item;
++
++	list_for_each_entry(group_item, &devlink->trap_group_list, list) {
++		if (group_item->group->id == id)
++			return group_item;
++	}
++
++	return NULL;
++}
++
++static struct devlink_trap_group_item *
++devlink_trap_group_item_get_from_info(struct devlink *devlink,
++				      struct genl_info *info)
++{
++	char *name;
++
++	if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
++		return NULL;
++	name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
++
++	return devlink_trap_group_item_lookup(devlink, name);
++}
++
++static int
++devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
++			   const struct devlink_trap_group_item *group_item,
++			   enum devlink_command cmd, u32 portid, u32 seq,
++			   int flags)
++{
++	void *hdr;
++	int err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
++			   group_item->group->name))
++		goto nla_put_failure;
++
++	if (group_item->group->generic &&
++	    nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
++		goto nla_put_failure;
++
++	if (group_item->policer_item &&
++	    nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
++			group_item->policer_item->policer->id))
++		goto nla_put_failure;
++
++	err = devlink_trap_group_stats_put(msg, group_item->stats);
++	if (err)
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
++					      struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_trap_group_item *group_item;
++	struct sk_buff *msg;
++	int err;
++
++	if (list_empty(&devlink->trap_group_list))
++		return -EOPNOTSUPP;
++
++	group_item = devlink_trap_group_item_get_from_info(devlink, info);
++	if (!group_item) {
++		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
++		return -ENOENT;
++	}
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_trap_group_fill(msg, devlink, group_item,
++					 DEVLINK_CMD_TRAP_GROUP_NEW,
++					 info->snd_portid, info->snd_seq, 0);
++	if (err)
++		goto err_trap_group_fill;
++
++	return genlmsg_reply(msg, info);
++
++err_trap_group_fill:
++	nlmsg_free(msg);
++	return err;
++}
++
++static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
++						struct netlink_callback *cb)
++{
++	enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
++	struct devlink_trap_group_item *group_item;
++	u32 portid = NETLINK_CB(cb->skb).portid;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(group_item, &devlink->trap_group_list,
++				    list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_trap_group_fill(msg, devlink,
++							 group_item, cmd,
++							 portid,
++							 cb->nlh->nlmsg_seq,
++							 NLM_F_MULTI);
++			if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int
++__devlink_trap_group_action_set(struct devlink *devlink,
++				struct devlink_trap_group_item *group_item,
++				enum devlink_trap_action trap_action,
++				struct netlink_ext_ack *extack)
++{
++	const char *group_name = group_item->group->name;
++	struct devlink_trap_item *trap_item;
++	int err;
++
++	if (devlink->ops->trap_group_action_set) {
++		err = devlink->ops->trap_group_action_set(devlink, group_item->group,
++							  trap_action, extack);
++		if (err)
++			return err;
++
++		list_for_each_entry(trap_item, &devlink->trap_list, list) {
++			if (strcmp(trap_item->group_item->group->name, group_name))
++				continue;
++			if (trap_item->action != trap_action &&
++			    trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
++				continue;
++			trap_item->action = trap_action;
++		}
++
++		return 0;
++	}
++
++	list_for_each_entry(trap_item, &devlink->trap_list, list) {
++		if (strcmp(trap_item->group_item->group->name, group_name))
++			continue;
++		err = __devlink_trap_action_set(devlink, trap_item,
++						trap_action, extack);
++		if (err)
++			return err;
++	}
++
++	return 0;
++}
++
++static int
++devlink_trap_group_action_set(struct devlink *devlink,
++			      struct devlink_trap_group_item *group_item,
++			      struct genl_info *info, bool *p_modified)
++{
++	enum devlink_trap_action trap_action;
++	int err;
++
++	if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
++		return 0;
++
++	err = devlink_trap_action_get_from_info(info, &trap_action);
++	if (err) {
++		NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
++		return -EINVAL;
++	}
++
++	err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
++					      info->extack);
++	if (err)
++		return err;
++
++	*p_modified = true;
++
++	return 0;
++}
++
++static int devlink_trap_group_set(struct devlink *devlink,
++				  struct devlink_trap_group_item *group_item,
++				  struct genl_info *info)
++{
++	struct devlink_trap_policer_item *policer_item;
++	struct netlink_ext_ack *extack = info->extack;
++	const struct devlink_trap_policer *policer;
++	struct nlattr **attrs = info->attrs;
++	int err;
++
++	if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
++		return 0;
++
++	if (!devlink->ops->trap_group_set)
++		return -EOPNOTSUPP;
++
++	policer_item = group_item->policer_item;
++	if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
++		u32 policer_id;
++
++		policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
++		policer_item = devlink_trap_policer_item_lookup(devlink,
++								policer_id);
++		if (policer_id && !policer_item) {
++			NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
++			return -ENOENT;
++		}
++	}
++	policer = policer_item ? policer_item->policer : NULL;
++
++	err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
++					   extack);
++	if (err)
++		return err;
++
++	group_item->policer_item = policer_item;
++
++	return 0;
++}
++
++static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
++					      struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++	struct devlink_trap_group_item *group_item;
++	bool modified = false;
++	int err;
++
++	if (list_empty(&devlink->trap_group_list))
++		return -EOPNOTSUPP;
++
++	group_item = devlink_trap_group_item_get_from_info(devlink, info);
++	if (!group_item) {
++		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
++		return -ENOENT;
++	}
++
++	err = devlink_trap_group_action_set(devlink, group_item, info,
++					    &modified);
++	if (err)
++		return err;
++
++	err = devlink_trap_group_set(devlink, group_item, info);
++	if (err)
++		goto err_trap_group_set;
++
++	return 0;
++
++err_trap_group_set:
++	if (modified)
++		NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
++	return err;
++}
++
++static struct devlink_trap_policer_item *
++devlink_trap_policer_item_get_from_info(struct devlink *devlink,
++					struct genl_info *info)
++{
++	u32 id;
++
++	if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
++		return NULL;
++	id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
++
++	return devlink_trap_policer_item_lookup(devlink, id);
++}
++
++static int
++devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
++			       const struct devlink_trap_policer *policer)
++{
++	struct nlattr *attr;
++	u64 drops;
++	int err;
++
++	if (!devlink->ops->trap_policer_counter_get)
++		return 0;
++
++	err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
++	if (err)
++		return err;
++
++	attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
++	if (!attr)
++		return -EMSGSIZE;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
++			      DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	nla_nest_end(msg, attr);
++
++	return 0;
++
++nla_put_failure:
++	nla_nest_cancel(msg, attr);
++	return -EMSGSIZE;
++}
++
++static int
++devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
++			     const struct devlink_trap_policer_item *policer_item,
++			     enum devlink_command cmd, u32 portid, u32 seq,
++			     int flags)
++{
++	void *hdr;
++	int err;
++
++	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
++	if (!hdr)
++		return -EMSGSIZE;
++
++	if (devlink_nl_put_handle(msg, devlink))
++		goto nla_put_failure;
++
++	if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
++			policer_item->policer->id))
++		goto nla_put_failure;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
++			      policer_item->rate, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
++			      policer_item->burst, DEVLINK_ATTR_PAD))
++		goto nla_put_failure;
++
++	err = devlink_trap_policer_stats_put(msg, devlink,
++					     policer_item->policer);
++	if (err)
++		goto nla_put_failure;
++
++	genlmsg_end(msg, hdr);
++
++	return 0;
++
++nla_put_failure:
++	genlmsg_cancel(msg, hdr);
++	return -EMSGSIZE;
++}
++
++static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
++						struct genl_info *info)
++{
++	struct devlink_trap_policer_item *policer_item;
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++	struct sk_buff *msg;
++	int err;
++
++	if (list_empty(&devlink->trap_policer_list))
++		return -EOPNOTSUPP;
++
++	policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
++	if (!policer_item) {
++		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
++		return -ENOENT;
++	}
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return -ENOMEM;
++
++	err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
++					   DEVLINK_CMD_TRAP_POLICER_NEW,
++					   info->snd_portid, info->snd_seq, 0);
++	if (err)
++		goto err_trap_policer_fill;
++
++	return genlmsg_reply(msg, info);
++
++err_trap_policer_fill:
++	nlmsg_free(msg);
++	return err;
++}
++
++static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
++						  struct netlink_callback *cb)
++{
++	enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
++	struct devlink_trap_policer_item *policer_item;
++	u32 portid = NETLINK_CB(cb->skb).portid;
++	struct devlink *devlink;
++	int start = cb->args[0];
++	unsigned long index;
++	int idx = 0;
++	int err;
++
++	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
++		devl_lock(devlink);
++		list_for_each_entry(policer_item, &devlink->trap_policer_list,
++				    list) {
++			if (idx < start) {
++				idx++;
++				continue;
++			}
++			err = devlink_nl_trap_policer_fill(msg, devlink,
++							   policer_item, cmd,
++							   portid,
++							   cb->nlh->nlmsg_seq,
++							   NLM_F_MULTI);
++			if (err) {
++				devl_unlock(devlink);
++				devlink_put(devlink);
++				goto out;
++			}
++			idx++;
++		}
++		devl_unlock(devlink);
++		devlink_put(devlink);
++	}
++out:
++	cb->args[0] = idx;
++	return msg->len;
++}
++
++static int
++devlink_trap_policer_set(struct devlink *devlink,
++			 struct devlink_trap_policer_item *policer_item,
++			 struct genl_info *info)
++{
++	struct netlink_ext_ack *extack = info->extack;
++	struct nlattr **attrs = info->attrs;
++	u64 rate, burst;
++	int err;
++
++	rate = policer_item->rate;
++	burst = policer_item->burst;
++
++	if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
++		rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
++
++	if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
++		burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
++
++	if (rate < policer_item->policer->min_rate) {
++		NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
++		return -EINVAL;
++	}
++
++	if (rate > policer_item->policer->max_rate) {
++		NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
++		return -EINVAL;
++	}
++
++	if (burst < policer_item->policer->min_burst) {
++		NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
++		return -EINVAL;
++	}
++
++	if (burst > policer_item->policer->max_burst) {
++		NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
++		return -EINVAL;
++	}
++
++	err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
++					     rate, burst, info->extack);
++	if (err)
++		return err;
++
++	policer_item->rate = rate;
++	policer_item->burst = burst;
++
++	return 0;
++}
++
++static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
++						struct genl_info *info)
++{
++	struct devlink_trap_policer_item *policer_item;
++	struct netlink_ext_ack *extack = info->extack;
++	struct devlink *devlink = info->user_ptr[0];
++
++	if (list_empty(&devlink->trap_policer_list))
++		return -EOPNOTSUPP;
++
++	if (!devlink->ops->trap_policer_set)
++		return -EOPNOTSUPP;
++
++	policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
++	if (!policer_item) {
++		NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
++		return -ENOENT;
++	}
++
++	return devlink_trap_policer_set(devlink, policer_item, info);
++}
++
++static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
++	[DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
++		DEVLINK_ATTR_TRAP_POLICER_ID },
++	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
++						    DEVLINK_PORT_TYPE_IB),
++	[DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
++	[DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
++	[DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
++						       DEVLINK_ESWITCH_MODE_SWITCHDEV),
++	[DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
++	[DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
++	[DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
++		NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
++	[DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
++	[DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
++	[DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
++							DEVLINK_RELOAD_ACTION_MAX),
++	[DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
++	[DEVLINK_ATTR_PORT_FLAVOUR] = { .type = NLA_U16 },
++	[DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 },
++	[DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 },
++	[DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 },
++	[DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_LINECARD_INDEX] = { .type = NLA_U32 },
++	[DEVLINK_ATTR_LINECARD_TYPE] = { .type = NLA_NUL_STRING },
++	[DEVLINK_ATTR_SELFTESTS] = { .type = NLA_NESTED },
++};
++
++static const struct genl_small_ops devlink_nl_ops[] = {
++	{
++		.cmd = DEVLINK_CMD_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_get_doit,
++		.dumpit = devlink_nl_cmd_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_port_get_doit,
++		.dumpit = devlink_nl_cmd_port_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_port_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_RATE_GET,
++		.doit = devlink_nl_cmd_rate_get_doit,
++		.dumpit = devlink_nl_cmd_rate_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_RATE_SET,
++		.doit = devlink_nl_cmd_rate_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
++	},
++	{
++		.cmd = DEVLINK_CMD_RATE_NEW,
++		.doit = devlink_nl_cmd_rate_new_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_RATE_DEL,
++		.doit = devlink_nl_cmd_rate_del_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE,
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_SPLIT,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_port_split_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_UNSPLIT,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_port_unsplit_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_NEW,
++		.doit = devlink_nl_cmd_port_new_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_DEL,
++		.doit = devlink_nl_cmd_port_del_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_LINECARD_GET,
++		.doit = devlink_nl_cmd_linecard_get_doit,
++		.dumpit = devlink_nl_cmd_linecard_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_LINECARD_SET,
++		.doit = devlink_nl_cmd_linecard_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_LINECARD,
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_get_doit,
++		.dumpit = devlink_nl_cmd_sb_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_POOL_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_pool_get_doit,
++		.dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_POOL_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_pool_set_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_port_pool_get_doit,
++		.dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_port_pool_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
++		.dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_occ_snapshot_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_sb_occ_max_clear_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_ESWITCH_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_eswitch_get_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_ESWITCH_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_eswitch_set_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_dpipe_table_get,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_dpipe_entries_get,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_dpipe_headers_get,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_dpipe_table_counters_set,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_RESOURCE_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_resource_set,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_RESOURCE_DUMP,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_resource_dump,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_RELOAD,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_reload,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_PARAM_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_param_get_doit,
++		.dumpit = devlink_nl_cmd_param_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_PARAM_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_param_set_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_PARAM_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_port_param_get_doit,
++		.dumpit = devlink_nl_cmd_port_param_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_PORT_PARAM_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_port_param_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_REGION_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_region_get_doit,
++		.dumpit = devlink_nl_cmd_region_get_dumpit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_REGION_NEW,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_region_new,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_REGION_DEL,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_region_del,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_REGION_READ,
++		.validate = GENL_DONT_VALIDATE_STRICT |
++			    GENL_DONT_VALIDATE_DUMP_STRICT,
++		.dumpit = devlink_nl_cmd_region_read_dumpit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_INFO_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_info_get_doit,
++		.dumpit = devlink_nl_cmd_info_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_health_reporter_get_doit,
++		.dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_health_reporter_set_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_health_reporter_recover_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_health_reporter_diagnose_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
++		.validate = GENL_DONT_VALIDATE_STRICT |
++			    GENL_DONT_VALIDATE_DUMP_STRICT,
++		.dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_health_reporter_test_doit,
++		.flags = GENL_ADMIN_PERM,
++		.internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT,
++	},
++	{
++		.cmd = DEVLINK_CMD_FLASH_UPDATE,
++		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
++		.doit = devlink_nl_cmd_flash_update,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_TRAP_GET,
++		.doit = devlink_nl_cmd_trap_get_doit,
++		.dumpit = devlink_nl_cmd_trap_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_TRAP_SET,
++		.doit = devlink_nl_cmd_trap_set_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_TRAP_GROUP_GET,
++		.doit = devlink_nl_cmd_trap_group_get_doit,
++		.dumpit = devlink_nl_cmd_trap_group_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_TRAP_GROUP_SET,
++		.doit = devlink_nl_cmd_trap_group_set_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_TRAP_POLICER_GET,
++		.doit = devlink_nl_cmd_trap_policer_get_doit,
++		.dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_TRAP_POLICER_SET,
++		.doit = devlink_nl_cmd_trap_policer_set_doit,
++		.flags = GENL_ADMIN_PERM,
++	},
++	{
++		.cmd = DEVLINK_CMD_SELFTESTS_GET,
++		.doit = devlink_nl_cmd_selftests_get_doit,
++		.dumpit = devlink_nl_cmd_selftests_get_dumpit
++		/* can be retrieved by unprivileged users */
++	},
++	{
++		.cmd = DEVLINK_CMD_SELFTESTS_RUN,
++		.doit = devlink_nl_cmd_selftests_run,
++		.flags = GENL_ADMIN_PERM,
++	},
++};
++
++static struct genl_family devlink_nl_family __ro_after_init = {
++	.name		= DEVLINK_GENL_NAME,
++	.version	= DEVLINK_GENL_VERSION,
++	.maxattr	= DEVLINK_ATTR_MAX,
++	.policy = devlink_nl_policy,
++	.netnsok	= true,
++	.parallel_ops	= true,
++	.pre_doit	= devlink_nl_pre_doit,
++	.post_doit	= devlink_nl_post_doit,
++	.module		= THIS_MODULE,
++	.small_ops	= devlink_nl_ops,
++	.n_small_ops	= ARRAY_SIZE(devlink_nl_ops),
++	.resv_start_op	= DEVLINK_CMD_SELFTESTS_RUN + 1,
++	.mcgrps		= devlink_nl_mcgrps,
++	.n_mcgrps	= ARRAY_SIZE(devlink_nl_mcgrps),
++};
++
++static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
++{
++	const struct devlink_reload_combination *comb;
++	int i;
++
++	if (!devlink_reload_supported(ops)) {
++		if (WARN_ON(ops->reload_actions))
++			return false;
++		return true;
++	}
++
++	if (WARN_ON(!ops->reload_actions ||
++		    ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
++		    ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
++		return false;
++
++	if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
++		    ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
++		return false;
++
++	for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
++		comb = &devlink_reload_invalid_combinations[i];
++		if (ops->reload_actions == BIT(comb->action) &&
++		    ops->reload_limits == BIT(comb->limit))
++			return false;
++	}
++	return true;
++}
++
++/**
++ *	devlink_set_features - Set devlink supported features
++ *
++ *	@devlink: devlink
++ *	@features: devlink support features
++ *
++ *	This interface allows us to set reload ops separatelly from
++ *	the devlink_alloc.
++ */
++void devlink_set_features(struct devlink *devlink, u64 features)
++{
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	WARN_ON(features & DEVLINK_F_RELOAD &&
++		!devlink_reload_supported(devlink->ops));
++	devlink->features = features;
++}
++EXPORT_SYMBOL_GPL(devlink_set_features);
++
++/**
++ *	devlink_alloc_ns - Allocate new devlink instance resources
++ *	in specific namespace
++ *
++ *	@ops: ops
++ *	@priv_size: size of user private data
++ *	@net: net namespace
++ *	@dev: parent device
++ *
++ *	Allocate new devlink instance resources, including devlink index
++ *	and name.
++ */
++struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
++				 size_t priv_size, struct net *net,
++				 struct device *dev)
++{
++	struct devlink *devlink;
++	static u32 last_id;
++	int ret;
++
++	WARN_ON(!ops || !dev);
++	if (!devlink_reload_actions_valid(ops))
++		return NULL;
++
++	devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
++	if (!devlink)
++		return NULL;
++
++	ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
++			      &last_id, GFP_KERNEL);
++	if (ret < 0) {
++		kfree(devlink);
++		return NULL;
++	}
++
++	devlink->dev = dev;
++	devlink->ops = ops;
++	xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
++	write_pnet(&devlink->_net, net);
++	INIT_LIST_HEAD(&devlink->port_list);
++	INIT_LIST_HEAD(&devlink->rate_list);
++	INIT_LIST_HEAD(&devlink->linecard_list);
++	INIT_LIST_HEAD(&devlink->sb_list);
++	INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
++	INIT_LIST_HEAD(&devlink->resource_list);
++	INIT_LIST_HEAD(&devlink->param_list);
++	INIT_LIST_HEAD(&devlink->region_list);
++	INIT_LIST_HEAD(&devlink->reporter_list);
++	INIT_LIST_HEAD(&devlink->trap_list);
++	INIT_LIST_HEAD(&devlink->trap_group_list);
++	INIT_LIST_HEAD(&devlink->trap_policer_list);
++	lockdep_register_key(&devlink->lock_key);
++	mutex_init(&devlink->lock);
++	lockdep_set_class(&devlink->lock, &devlink->lock_key);
++	mutex_init(&devlink->reporters_lock);
++	mutex_init(&devlink->linecards_lock);
++	refcount_set(&devlink->refcount, 1);
++	init_completion(&devlink->comp);
++
++	return devlink;
++}
++EXPORT_SYMBOL_GPL(devlink_alloc_ns);
++
++static void
++devlink_trap_policer_notify(struct devlink *devlink,
++			    const struct devlink_trap_policer_item *policer_item,
++			    enum devlink_command cmd);
++static void
++devlink_trap_group_notify(struct devlink *devlink,
++			  const struct devlink_trap_group_item *group_item,
++			  enum devlink_command cmd);
++static void devlink_trap_notify(struct devlink *devlink,
++				const struct devlink_trap_item *trap_item,
++				enum devlink_command cmd);
++
++static void devlink_notify_register(struct devlink *devlink)
++{
++	struct devlink_trap_policer_item *policer_item;
++	struct devlink_trap_group_item *group_item;
++	struct devlink_param_item *param_item;
++	struct devlink_trap_item *trap_item;
++	struct devlink_port *devlink_port;
++	struct devlink_linecard *linecard;
++	struct devlink_rate *rate_node;
++	struct devlink_region *region;
++
++	devlink_notify(devlink, DEVLINK_CMD_NEW);
++	list_for_each_entry(linecard, &devlink->linecard_list, list)
++		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++
++	list_for_each_entry(devlink_port, &devlink->port_list, list)
++		devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
++
++	list_for_each_entry(policer_item, &devlink->trap_policer_list, list)
++		devlink_trap_policer_notify(devlink, policer_item,
++					    DEVLINK_CMD_TRAP_POLICER_NEW);
++
++	list_for_each_entry(group_item, &devlink->trap_group_list, list)
++		devlink_trap_group_notify(devlink, group_item,
++					  DEVLINK_CMD_TRAP_GROUP_NEW);
++
++	list_for_each_entry(trap_item, &devlink->trap_list, list)
++		devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
++
++	list_for_each_entry(rate_node, &devlink->rate_list, list)
++		devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
++
++	list_for_each_entry(region, &devlink->region_list, list)
++		devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
++
++	list_for_each_entry(param_item, &devlink->param_list, list)
++		devlink_param_notify(devlink, 0, param_item,
++				     DEVLINK_CMD_PARAM_NEW);
++}
++
++static void devlink_notify_unregister(struct devlink *devlink)
++{
++	struct devlink_trap_policer_item *policer_item;
++	struct devlink_trap_group_item *group_item;
++	struct devlink_param_item *param_item;
++	struct devlink_trap_item *trap_item;
++	struct devlink_port *devlink_port;
++	struct devlink_linecard *linecard;
++	struct devlink_rate *rate_node;
++	struct devlink_region *region;
++
++	list_for_each_entry_reverse(param_item, &devlink->param_list, list)
++		devlink_param_notify(devlink, 0, param_item,
++				     DEVLINK_CMD_PARAM_DEL);
++
++	list_for_each_entry_reverse(region, &devlink->region_list, list)
++		devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
++
++	list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
++		devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
++
++	list_for_each_entry_reverse(trap_item, &devlink->trap_list, list)
++		devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
++
++	list_for_each_entry_reverse(group_item, &devlink->trap_group_list, list)
++		devlink_trap_group_notify(devlink, group_item,
++					  DEVLINK_CMD_TRAP_GROUP_DEL);
++	list_for_each_entry_reverse(policer_item, &devlink->trap_policer_list,
++				    list)
++		devlink_trap_policer_notify(devlink, policer_item,
++					    DEVLINK_CMD_TRAP_POLICER_DEL);
++
++	list_for_each_entry_reverse(devlink_port, &devlink->port_list, list)
++		devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
++	list_for_each_entry_reverse(linecard, &devlink->linecard_list, list)
++		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_DEL);
++	devlink_notify(devlink, DEVLINK_CMD_DEL);
++}
++
++/**
++ *	devlink_register - Register devlink instance
++ *
++ *	@devlink: devlink
++ */
++void devlink_register(struct devlink *devlink)
++{
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++	/* Make sure that we are in .probe() routine */
++
++	xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
++	devlink_notify_register(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_register);
++
++/**
++ *	devlink_unregister - Unregister devlink instance
++ *
++ *	@devlink: devlink
++ */
++void devlink_unregister(struct devlink *devlink)
++{
++	ASSERT_DEVLINK_REGISTERED(devlink);
++	/* Make sure that we are in .remove() routine */
++
++	xa_set_mark(&devlinks, devlink->index, DEVLINK_UNREGISTERING);
++	devlink_put(devlink);
++	wait_for_completion(&devlink->comp);
++
++	devlink_notify_unregister(devlink);
++	xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
++	xa_clear_mark(&devlinks, devlink->index, DEVLINK_UNREGISTERING);
++}
++EXPORT_SYMBOL_GPL(devlink_unregister);
++
++/**
++ *	devlink_free - Free devlink instance resources
++ *
++ *	@devlink: devlink
++ */
++void devlink_free(struct devlink *devlink)
++{
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	mutex_destroy(&devlink->linecards_lock);
++	mutex_destroy(&devlink->reporters_lock);
++	mutex_destroy(&devlink->lock);
++	lockdep_unregister_key(&devlink->lock_key);
++	WARN_ON(!list_empty(&devlink->trap_policer_list));
++	WARN_ON(!list_empty(&devlink->trap_group_list));
++	WARN_ON(!list_empty(&devlink->trap_list));
++	WARN_ON(!list_empty(&devlink->reporter_list));
++	WARN_ON(!list_empty(&devlink->region_list));
++	WARN_ON(!list_empty(&devlink->param_list));
++	WARN_ON(!list_empty(&devlink->resource_list));
++	WARN_ON(!list_empty(&devlink->dpipe_table_list));
++	WARN_ON(!list_empty(&devlink->sb_list));
++	WARN_ON(!list_empty(&devlink->rate_list));
++	WARN_ON(!list_empty(&devlink->linecard_list));
++	WARN_ON(!list_empty(&devlink->port_list));
++
++	xa_destroy(&devlink->snapshot_ids);
++	xa_erase(&devlinks, devlink->index);
++
++	kfree(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_free);
++
++static void devlink_port_type_warn(struct work_struct *work)
++{
++	struct devlink_port *port = container_of(to_delayed_work(work),
++						 struct devlink_port,
++						 type_warn_dw);
++	dev_warn(port->devlink->dev, "Type was not set for devlink port.");
++}
++
++static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
++{
++	/* Ignore CPU and DSA flavours. */
++	return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
++	       devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
++	       devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
++}
++
++#define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
++
++static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
++{
++	if (!devlink_port_type_should_warn(devlink_port))
++		return;
++	/* Schedule a work to WARN in case driver does not set port
++	 * type within timeout.
++	 */
++	schedule_delayed_work(&devlink_port->type_warn_dw,
++			      DEVLINK_PORT_TYPE_WARN_TIMEOUT);
++}
++
++static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
++{
++	if (!devlink_port_type_should_warn(devlink_port))
++		return;
++	cancel_delayed_work_sync(&devlink_port->type_warn_dw);
++}
++
++/**
++ * devlink_port_init() - Init devlink port
++ *
++ * @devlink: devlink
++ * @devlink_port: devlink port
++ *
++ * Initialize essencial stuff that is needed for functions
++ * that may be called before devlink port registration.
++ * Call to this function is optional and not needed
++ * in case the driver does not use such functions.
++ */
++void devlink_port_init(struct devlink *devlink,
++		       struct devlink_port *devlink_port)
++{
++	if (devlink_port->initialized)
++		return;
++	devlink_port->devlink = devlink;
++	INIT_LIST_HEAD(&devlink_port->region_list);
++	devlink_port->initialized = true;
++}
++EXPORT_SYMBOL_GPL(devlink_port_init);
++
++/**
++ * devlink_port_fini() - Deinitialize devlink port
++ *
++ * @devlink_port: devlink port
++ *
++ * Deinitialize essencial stuff that is in use for functions
++ * that may be called after devlink port unregistration.
++ * Call to this function is optional and not needed
++ * in case the driver does not use such functions.
++ */
++void devlink_port_fini(struct devlink_port *devlink_port)
++{
++	WARN_ON(!list_empty(&devlink_port->region_list));
++}
++EXPORT_SYMBOL_GPL(devlink_port_fini);
++
++/**
++ * devl_port_register() - Register devlink port
++ *
++ * @devlink: devlink
++ * @devlink_port: devlink port
++ * @port_index: driver-specific numerical identifier of the port
++ *
++ * Register devlink port with provided port index. User can use
++ * any indexing, even hw-related one. devlink_port structure
++ * is convenient to be embedded inside user driver private structure.
++ * Note that the caller should take care of zeroing the devlink_port
++ * structure.
++ */
++int devl_port_register(struct devlink *devlink,
++		       struct devlink_port *devlink_port,
++		       unsigned int port_index)
++{
++	devl_assert_locked(devlink);
++
++	if (devlink_port_index_exists(devlink, port_index))
++		return -EEXIST;
++
++	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
++
++	devlink_port_init(devlink, devlink_port);
++	devlink_port->registered = true;
++	devlink_port->index = port_index;
++	spin_lock_init(&devlink_port->type_lock);
++	INIT_LIST_HEAD(&devlink_port->reporter_list);
++	mutex_init(&devlink_port->reporters_lock);
++	list_add_tail(&devlink_port->list, &devlink->port_list);
++
++	INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
++	devlink_port_type_warn_schedule(devlink_port);
++	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_port_register);
++
++/**
++ *	devlink_port_register - Register devlink port
++ *
++ *	@devlink: devlink
++ *	@devlink_port: devlink port
++ *	@port_index: driver-specific numerical identifier of the port
++ *
++ *	Register devlink port with provided port index. User can use
++ *	any indexing, even hw-related one. devlink_port structure
++ *	is convenient to be embedded inside user driver private structure.
++ *	Note that the caller should take care of zeroing the devlink_port
++ *	structure.
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++int devlink_port_register(struct devlink *devlink,
++			  struct devlink_port *devlink_port,
++			  unsigned int port_index)
++{
++	int err;
++
++	devl_lock(devlink);
++	err = devl_port_register(devlink, devlink_port, port_index);
++	devl_unlock(devlink);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_port_register);
++
++/**
++ * devl_port_unregister() - Unregister devlink port
++ *
++ * @devlink_port: devlink port
++ */
++void devl_port_unregister(struct devlink_port *devlink_port)
++{
++	lockdep_assert_held(&devlink_port->devlink->lock);
++
++	devlink_port_type_warn_cancel(devlink_port);
++	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
++	list_del(&devlink_port->list);
++	WARN_ON(!list_empty(&devlink_port->reporter_list));
++	mutex_destroy(&devlink_port->reporters_lock);
++	devlink_port->registered = false;
++}
++EXPORT_SYMBOL_GPL(devl_port_unregister);
++
++/**
++ *	devlink_port_unregister - Unregister devlink port
++ *
++ *	@devlink_port: devlink port
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_port_unregister(struct devlink_port *devlink_port)
++{
++	struct devlink *devlink = devlink_port->devlink;
++
++	devl_lock(devlink);
++	devl_port_unregister(devlink_port);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_port_unregister);
++
++static void __devlink_port_type_set(struct devlink_port *devlink_port,
++				    enum devlink_port_type type,
++				    void *type_dev)
++{
++	ASSERT_DEVLINK_PORT_REGISTERED(devlink_port);
++
++	devlink_port_type_warn_cancel(devlink_port);
++	spin_lock_bh(&devlink_port->type_lock);
++	devlink_port->type = type;
++	devlink_port->type_dev = type_dev;
++	spin_unlock_bh(&devlink_port->type_lock);
++	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
++}
++
++static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
++					    struct net_device *netdev)
++{
++	const struct net_device_ops *ops = netdev->netdev_ops;
++
++	/* If driver registers devlink port, it should set devlink port
++	 * attributes accordingly so the compat functions are called
++	 * and the original ops are not used.
++	 */
++	if (ops->ndo_get_phys_port_name) {
++		/* Some drivers use the same set of ndos for netdevs
++		 * that have devlink_port registered and also for
++		 * those who don't. Make sure that ndo_get_phys_port_name
++		 * returns -EOPNOTSUPP here in case it is defined.
++		 * Warn if not.
++		 */
++		char name[IFNAMSIZ];
++		int err;
++
++		err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
++		WARN_ON(err != -EOPNOTSUPP);
++	}
++	if (ops->ndo_get_port_parent_id) {
++		/* Some drivers use the same set of ndos for netdevs
++		 * that have devlink_port registered and also for
++		 * those who don't. Make sure that ndo_get_port_parent_id
++		 * returns -EOPNOTSUPP here in case it is defined.
++		 * Warn if not.
++		 */
++		struct netdev_phys_item_id ppid;
++		int err;
++
++		err = ops->ndo_get_port_parent_id(netdev, &ppid);
++		WARN_ON(err != -EOPNOTSUPP);
++	}
++}
++
++/**
++ *	devlink_port_type_eth_set - Set port type to Ethernet
++ *
++ *	@devlink_port: devlink port
++ *	@netdev: related netdevice
++ */
++void devlink_port_type_eth_set(struct devlink_port *devlink_port,
++			       struct net_device *netdev)
++{
++	if (netdev)
++		devlink_port_type_netdev_checks(devlink_port, netdev);
++	else
++		dev_warn(devlink_port->devlink->dev,
++			 "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
++			 devlink_port->index);
++
++	__devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
++}
++EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
++
++/**
++ *	devlink_port_type_ib_set - Set port type to InfiniBand
++ *
++ *	@devlink_port: devlink port
++ *	@ibdev: related IB device
++ */
++void devlink_port_type_ib_set(struct devlink_port *devlink_port,
++			      struct ib_device *ibdev)
++{
++	__devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
++}
++EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
++
++/**
++ *	devlink_port_type_clear - Clear port type
++ *
++ *	@devlink_port: devlink port
++ */
++void devlink_port_type_clear(struct devlink_port *devlink_port)
++{
++	__devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
++	devlink_port_type_warn_schedule(devlink_port);
++}
++EXPORT_SYMBOL_GPL(devlink_port_type_clear);
++
++static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
++				    enum devlink_port_flavour flavour)
++{
++	struct devlink_port_attrs *attrs = &devlink_port->attrs;
++
++	devlink_port->attrs_set = true;
++	attrs->flavour = flavour;
++	if (attrs->switch_id.id_len) {
++		devlink_port->switch_port = true;
++		if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
++			attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
++	} else {
++		devlink_port->switch_port = false;
++	}
++	return 0;
++}
++
++/**
++ *	devlink_port_attrs_set - Set port attributes
++ *
++ *	@devlink_port: devlink port
++ *	@attrs: devlink port attrs
++ */
++void devlink_port_attrs_set(struct devlink_port *devlink_port,
++			    struct devlink_port_attrs *attrs)
++{
++	int ret;
++
++	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
++
++	devlink_port->attrs = *attrs;
++	ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
++	if (ret)
++		return;
++	WARN_ON(attrs->splittable && attrs->split);
++}
++EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
++
++/**
++ *	devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
++ *
++ *	@devlink_port: devlink port
++ *	@controller: associated controller number for the devlink port instance
++ *	@pf: associated PF for the devlink port instance
++ *	@external: indicates if the port is for an external controller
++ */
++void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
++				   u16 pf, bool external)
++{
++	struct devlink_port_attrs *attrs = &devlink_port->attrs;
++	int ret;
++
++	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
++
++	ret = __devlink_port_attrs_set(devlink_port,
++				       DEVLINK_PORT_FLAVOUR_PCI_PF);
++	if (ret)
++		return;
++	attrs->pci_pf.controller = controller;
++	attrs->pci_pf.pf = pf;
++	attrs->pci_pf.external = external;
++}
++EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
++
++/**
++ *	devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
++ *
++ *	@devlink_port: devlink port
++ *	@controller: associated controller number for the devlink port instance
++ *	@pf: associated PF for the devlink port instance
++ *	@vf: associated VF of a PF for the devlink port instance
++ *	@external: indicates if the port is for an external controller
++ */
++void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
++				   u16 pf, u16 vf, bool external)
++{
++	struct devlink_port_attrs *attrs = &devlink_port->attrs;
++	int ret;
++
++	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
++
++	ret = __devlink_port_attrs_set(devlink_port,
++				       DEVLINK_PORT_FLAVOUR_PCI_VF);
++	if (ret)
++		return;
++	attrs->pci_vf.controller = controller;
++	attrs->pci_vf.pf = pf;
++	attrs->pci_vf.vf = vf;
++	attrs->pci_vf.external = external;
++}
++EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
++
++/**
++ *	devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
++ *
++ *	@devlink_port: devlink port
++ *	@controller: associated controller number for the devlink port instance
++ *	@pf: associated PF for the devlink port instance
++ *	@sf: associated SF of a PF for the devlink port instance
++ *	@external: indicates if the port is for an external controller
++ */
++void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
++				   u16 pf, u32 sf, bool external)
++{
++	struct devlink_port_attrs *attrs = &devlink_port->attrs;
++	int ret;
++
++	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
++
++	ret = __devlink_port_attrs_set(devlink_port,
++				       DEVLINK_PORT_FLAVOUR_PCI_SF);
++	if (ret)
++		return;
++	attrs->pci_sf.controller = controller;
++	attrs->pci_sf.pf = pf;
++	attrs->pci_sf.sf = sf;
++	attrs->pci_sf.external = external;
++}
++EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);
++
++/**
++ * devl_rate_leaf_create - create devlink rate leaf
++ * @devlink_port: devlink port object to create rate object on
++ * @priv: driver private data
++ *
++ * Create devlink rate object of type leaf on provided @devlink_port.
++ */
++int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
++{
++	struct devlink *devlink = devlink_port->devlink;
++	struct devlink_rate *devlink_rate;
++
++	devl_assert_locked(devlink_port->devlink);
++
++	if (WARN_ON(devlink_port->devlink_rate))
++		return -EBUSY;
++
++	devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL);
++	if (!devlink_rate)
++		return -ENOMEM;
++
++	devlink_rate->type = DEVLINK_RATE_TYPE_LEAF;
++	devlink_rate->devlink = devlink;
++	devlink_rate->devlink_port = devlink_port;
++	devlink_rate->priv = priv;
++	list_add_tail(&devlink_rate->list, &devlink->rate_list);
++	devlink_port->devlink_rate = devlink_rate;
++	devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_rate_leaf_create);
++
++/**
++ * devl_rate_leaf_destroy - destroy devlink rate leaf
++ *
++ * @devlink_port: devlink port linked to the rate object
++ *
++ * Destroy the devlink rate object of type leaf on provided @devlink_port.
++ */
++void devl_rate_leaf_destroy(struct devlink_port *devlink_port)
++{
++	struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
++
++	devl_assert_locked(devlink_port->devlink);
++	if (!devlink_rate)
++		return;
++
++	devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
++	if (devlink_rate->parent)
++		refcount_dec(&devlink_rate->parent->refcnt);
++	list_del(&devlink_rate->list);
++	devlink_port->devlink_rate = NULL;
++	kfree(devlink_rate);
++}
++EXPORT_SYMBOL_GPL(devl_rate_leaf_destroy);
++
++/**
++ * devl_rate_nodes_destroy - destroy all devlink rate nodes on device
++ * @devlink: devlink instance
++ *
++ * Unset parent for all rate objects and destroy all rate nodes
++ * on specified device.
++ */
++void devl_rate_nodes_destroy(struct devlink *devlink)
++{
++	static struct devlink_rate *devlink_rate, *tmp;
++	const struct devlink_ops *ops = devlink->ops;
++
++	devl_assert_locked(devlink);
++
++	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
++		if (!devlink_rate->parent)
++			continue;
++
++		refcount_dec(&devlink_rate->parent->refcnt);
++		if (devlink_rate_is_leaf(devlink_rate))
++			ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
++						  NULL, NULL);
++		else if (devlink_rate_is_node(devlink_rate))
++			ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv,
++						  NULL, NULL);
++	}
++	list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
++		if (devlink_rate_is_node(devlink_rate)) {
++			ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
++			list_del(&devlink_rate->list);
++			kfree(devlink_rate->name);
++			kfree(devlink_rate);
++		}
++	}
++}
++EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy);
++
++/**
++ *	devlink_port_linecard_set - Link port with a linecard
++ *
++ *	@devlink_port: devlink port
++ *	@linecard: devlink linecard
++ */
++void devlink_port_linecard_set(struct devlink_port *devlink_port,
++			       struct devlink_linecard *linecard)
++{
++	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
++
++	devlink_port->linecard = linecard;
++}
++EXPORT_SYMBOL_GPL(devlink_port_linecard_set);
++
++static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
++					     char *name, size_t len)
++{
++	struct devlink_port_attrs *attrs = &devlink_port->attrs;
++	int n = 0;
++
++	if (!devlink_port->attrs_set)
++		return -EOPNOTSUPP;
++
++	switch (attrs->flavour) {
++	case DEVLINK_PORT_FLAVOUR_PHYSICAL:
++		if (devlink_port->linecard)
++			n = snprintf(name, len, "l%u",
++				     devlink_port->linecard->index);
++		if (n < len)
++			n += snprintf(name + n, len - n, "p%u",
++				      attrs->phys.port_number);
++		if (n < len && attrs->split)
++			n += snprintf(name + n, len - n, "s%u",
++				      attrs->phys.split_subport_number);
++		break;
++	case DEVLINK_PORT_FLAVOUR_CPU:
++	case DEVLINK_PORT_FLAVOUR_DSA:
++	case DEVLINK_PORT_FLAVOUR_UNUSED:
++		/* As CPU and DSA ports do not have a netdevice associated
++		 * case should not ever happen.
++		 */
++		WARN_ON(1);
++		return -EINVAL;
++	case DEVLINK_PORT_FLAVOUR_PCI_PF:
++		if (attrs->pci_pf.external) {
++			n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
++			if (n >= len)
++				return -EINVAL;
++			len -= n;
++			name += n;
++		}
++		n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
++		break;
++	case DEVLINK_PORT_FLAVOUR_PCI_VF:
++		if (attrs->pci_vf.external) {
++			n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
++			if (n >= len)
++				return -EINVAL;
++			len -= n;
++			name += n;
++		}
++		n = snprintf(name, len, "pf%uvf%u",
++			     attrs->pci_vf.pf, attrs->pci_vf.vf);
++		break;
++	case DEVLINK_PORT_FLAVOUR_PCI_SF:
++		if (attrs->pci_sf.external) {
++			n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
++			if (n >= len)
++				return -EINVAL;
++			len -= n;
++			name += n;
++		}
++		n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
++			     attrs->pci_sf.sf);
++		break;
++	case DEVLINK_PORT_FLAVOUR_VIRTUAL:
++		return -EOPNOTSUPP;
++	}
++
++	if (n >= len)
++		return -EINVAL;
++
++	return 0;
++}
++
++static int devlink_linecard_types_init(struct devlink_linecard *linecard)
++{
++	struct devlink_linecard_type *linecard_type;
++	unsigned int count;
++	int i;
++
++	count = linecard->ops->types_count(linecard, linecard->priv);
++	linecard->types = kmalloc_array(count, sizeof(*linecard_type),
++					GFP_KERNEL);
++	if (!linecard->types)
++		return -ENOMEM;
++	linecard->types_count = count;
++
++	for (i = 0; i < count; i++) {
++		linecard_type = &linecard->types[i];
++		linecard->ops->types_get(linecard, linecard->priv, i,
++					 &linecard_type->type,
++					 &linecard_type->priv);
++	}
++	return 0;
++}
++
++static void devlink_linecard_types_fini(struct devlink_linecard *linecard)
++{
++	kfree(linecard->types);
++}
++
++/**
++ *	devlink_linecard_create - Create devlink linecard
++ *
++ *	@devlink: devlink
++ *	@linecard_index: driver-specific numerical identifier of the linecard
++ *	@ops: linecards ops
++ *	@priv: user priv pointer
++ *
++ *	Create devlink linecard instance with provided linecard index.
++ *	Caller can use any indexing, even hw-related one.
++ *
++ *	Return: Line card structure or an ERR_PTR() encoded error code.
++ */
++struct devlink_linecard *
++devlink_linecard_create(struct devlink *devlink, unsigned int linecard_index,
++			const struct devlink_linecard_ops *ops, void *priv)
++{
++	struct devlink_linecard *linecard;
++	int err;
++
++	if (WARN_ON(!ops || !ops->provision || !ops->unprovision ||
++		    !ops->types_count || !ops->types_get))
++		return ERR_PTR(-EINVAL);
++
++	mutex_lock(&devlink->linecards_lock);
++	if (devlink_linecard_index_exists(devlink, linecard_index)) {
++		mutex_unlock(&devlink->linecards_lock);
++		return ERR_PTR(-EEXIST);
++	}
++
++	linecard = kzalloc(sizeof(*linecard), GFP_KERNEL);
++	if (!linecard) {
++		mutex_unlock(&devlink->linecards_lock);
++		return ERR_PTR(-ENOMEM);
++	}
++
++	linecard->devlink = devlink;
++	linecard->index = linecard_index;
++	linecard->ops = ops;
++	linecard->priv = priv;
++	linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
++	mutex_init(&linecard->state_lock);
++
++	err = devlink_linecard_types_init(linecard);
++	if (err) {
++		mutex_destroy(&linecard->state_lock);
++		kfree(linecard);
++		mutex_unlock(&devlink->linecards_lock);
++		return ERR_PTR(err);
++	}
++
++	list_add_tail(&linecard->list, &devlink->linecard_list);
++	refcount_set(&linecard->refcount, 1);
++	mutex_unlock(&devlink->linecards_lock);
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	return linecard;
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_create);
++
++/**
++ *	devlink_linecard_destroy - Destroy devlink linecard
++ *
++ *	@linecard: devlink linecard
++ */
++void devlink_linecard_destroy(struct devlink_linecard *linecard)
++{
++	struct devlink *devlink = linecard->devlink;
++
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_DEL);
++	mutex_lock(&devlink->linecards_lock);
++	list_del(&linecard->list);
++	devlink_linecard_types_fini(linecard);
++	mutex_unlock(&devlink->linecards_lock);
++	devlink_linecard_put(linecard);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_destroy);
++
++/**
++ *	devlink_linecard_provision_set - Set provisioning on linecard
++ *
++ *	@linecard: devlink linecard
++ *	@type: linecard type
++ *
++ *	This is either called directly from the provision() op call or
++ *	as a result of the provision() op call asynchronously.
++ */
++void devlink_linecard_provision_set(struct devlink_linecard *linecard,
++				    const char *type)
++{
++	mutex_lock(&linecard->state_lock);
++	WARN_ON(linecard->type && strcmp(linecard->type, type));
++	linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
++	linecard->type = type;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_provision_set);
++
++/**
++ *	devlink_linecard_provision_clear - Clear provisioning on linecard
++ *
++ *	@linecard: devlink linecard
++ *
++ *	This is either called directly from the unprovision() op call or
++ *	as a result of the unprovision() op call asynchronously.
++ */
++void devlink_linecard_provision_clear(struct devlink_linecard *linecard)
++{
++	mutex_lock(&linecard->state_lock);
++	WARN_ON(linecard->nested_devlink);
++	linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED;
++	linecard->type = NULL;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear);
++
++/**
++ *	devlink_linecard_provision_fail - Fail provisioning on linecard
++ *
++ *	@linecard: devlink linecard
++ *
++ *	This is either called directly from the provision() op call or
++ *	as a result of the provision() op call asynchronously.
++ */
++void devlink_linecard_provision_fail(struct devlink_linecard *linecard)
++{
++	mutex_lock(&linecard->state_lock);
++	WARN_ON(linecard->nested_devlink);
++	linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING_FAILED;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_provision_fail);
++
++/**
++ *	devlink_linecard_activate - Set linecard active
++ *
++ *	@linecard: devlink linecard
++ */
++void devlink_linecard_activate(struct devlink_linecard *linecard)
++{
++	mutex_lock(&linecard->state_lock);
++	WARN_ON(linecard->state != DEVLINK_LINECARD_STATE_PROVISIONED);
++	linecard->state = DEVLINK_LINECARD_STATE_ACTIVE;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_activate);
++
++/**
++ *	devlink_linecard_deactivate - Set linecard inactive
++ *
++ *	@linecard: devlink linecard
++ */
++void devlink_linecard_deactivate(struct devlink_linecard *linecard)
++{
++	mutex_lock(&linecard->state_lock);
++	switch (linecard->state) {
++	case DEVLINK_LINECARD_STATE_ACTIVE:
++		linecard->state = DEVLINK_LINECARD_STATE_PROVISIONED;
++		devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++		break;
++	case DEVLINK_LINECARD_STATE_UNPROVISIONING:
++		/* Line card is being deactivated as part
++		 * of unprovisioning flow.
++		 */
++		break;
++	default:
++		WARN_ON(1);
++		break;
++	}
++	mutex_unlock(&linecard->state_lock);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_deactivate);
++
++/**
++ *	devlink_linecard_nested_dl_set - Attach/detach nested devlink
++ *					 instance to linecard.
++ *
++ *	@linecard: devlink linecard
++ *	@nested_devlink: devlink instance to attach or NULL to detach
++ */
++void devlink_linecard_nested_dl_set(struct devlink_linecard *linecard,
++				    struct devlink *nested_devlink)
++{
++	mutex_lock(&linecard->state_lock);
++	linecard->nested_devlink = nested_devlink;
++	devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW);
++	mutex_unlock(&linecard->state_lock);
++}
++EXPORT_SYMBOL_GPL(devlink_linecard_nested_dl_set);
++
++int devl_sb_register(struct devlink *devlink, unsigned int sb_index,
++		     u32 size, u16 ingress_pools_count,
++		     u16 egress_pools_count, u16 ingress_tc_count,
++		     u16 egress_tc_count)
++{
++	struct devlink_sb *devlink_sb;
++
++	lockdep_assert_held(&devlink->lock);
++
++	if (devlink_sb_index_exists(devlink, sb_index))
++		return -EEXIST;
++
++	devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
++	if (!devlink_sb)
++		return -ENOMEM;
++	devlink_sb->index = sb_index;
++	devlink_sb->size = size;
++	devlink_sb->ingress_pools_count = ingress_pools_count;
++	devlink_sb->egress_pools_count = egress_pools_count;
++	devlink_sb->ingress_tc_count = ingress_tc_count;
++	devlink_sb->egress_tc_count = egress_tc_count;
++	list_add_tail(&devlink_sb->list, &devlink->sb_list);
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_sb_register);
++
++int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
++			u32 size, u16 ingress_pools_count,
++			u16 egress_pools_count, u16 ingress_tc_count,
++			u16 egress_tc_count)
++{
++	int err;
++
++	devl_lock(devlink);
++	err = devl_sb_register(devlink, sb_index, size, ingress_pools_count,
++			       egress_pools_count, ingress_tc_count,
++			       egress_tc_count);
++	devl_unlock(devlink);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_sb_register);
++
++void devl_sb_unregister(struct devlink *devlink, unsigned int sb_index)
++{
++	struct devlink_sb *devlink_sb;
++
++	lockdep_assert_held(&devlink->lock);
++
++	devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
++	WARN_ON(!devlink_sb);
++	list_del(&devlink_sb->list);
++	kfree(devlink_sb);
++}
++EXPORT_SYMBOL_GPL(devl_sb_unregister);
++
++void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
++{
++	devl_lock(devlink);
++	devl_sb_unregister(devlink, sb_index);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_sb_unregister);
++
++/**
++ * devl_dpipe_headers_register - register dpipe headers
++ *
++ * @devlink: devlink
++ * @dpipe_headers: dpipe header array
++ *
++ * Register the headers supported by hardware.
++ */
++void devl_dpipe_headers_register(struct devlink *devlink,
++				 struct devlink_dpipe_headers *dpipe_headers)
++{
++	lockdep_assert_held(&devlink->lock);
++
++	devlink->dpipe_headers = dpipe_headers;
++}
++EXPORT_SYMBOL_GPL(devl_dpipe_headers_register);
++
++/**
++ * devl_dpipe_headers_unregister - unregister dpipe headers
++ *
++ * @devlink: devlink
++ *
++ * Unregister the headers supported by hardware.
++ */
++void devl_dpipe_headers_unregister(struct devlink *devlink)
++{
++	lockdep_assert_held(&devlink->lock);
++
++	devlink->dpipe_headers = NULL;
++}
++EXPORT_SYMBOL_GPL(devl_dpipe_headers_unregister);
++
++/**
++ *	devlink_dpipe_table_counter_enabled - check if counter allocation
++ *					      required
++ *	@devlink: devlink
++ *	@table_name: tables name
++ *
++ *	Used by driver to check if counter allocation is required.
++ *	After counter allocation is turned on the table entries
++ *	are updated to include counter statistics.
++ *
++ *	After that point on the driver must respect the counter
++ *	state so that each entry added to the table is added
++ *	with a counter.
++ */
++bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
++					 const char *table_name)
++{
++	struct devlink_dpipe_table *table;
++	bool enabled;
++
++	rcu_read_lock();
++	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
++					 table_name, devlink);
++	enabled = false;
++	if (table)
++		enabled = table->counters_enabled;
++	rcu_read_unlock();
++	return enabled;
++}
++EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
++
++/**
++ * devl_dpipe_table_register - register dpipe table
++ *
++ * @devlink: devlink
++ * @table_name: table name
++ * @table_ops: table ops
++ * @priv: priv
++ * @counter_control_extern: external control for counters
++ */
++int devl_dpipe_table_register(struct devlink *devlink,
++			      const char *table_name,
++			      struct devlink_dpipe_table_ops *table_ops,
++			      void *priv, bool counter_control_extern)
++{
++	struct devlink_dpipe_table *table;
++
++	lockdep_assert_held(&devlink->lock);
++
++	if (WARN_ON(!table_ops->size_get))
++		return -EINVAL;
++
++	if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
++				     devlink))
++		return -EEXIST;
++
++	table = kzalloc(sizeof(*table), GFP_KERNEL);
++	if (!table)
++		return -ENOMEM;
++
++	table->name = table_name;
++	table->table_ops = table_ops;
++	table->priv = priv;
++	table->counter_control_extern = counter_control_extern;
++
++	list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_dpipe_table_register);
++
++/**
++ * devl_dpipe_table_unregister - unregister dpipe table
++ *
++ * @devlink: devlink
++ * @table_name: table name
++ */
++void devl_dpipe_table_unregister(struct devlink *devlink,
++				 const char *table_name)
++{
++	struct devlink_dpipe_table *table;
++
++	lockdep_assert_held(&devlink->lock);
++
++	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
++					 table_name, devlink);
++	if (!table)
++		return;
++	list_del_rcu(&table->list);
++	kfree_rcu(table, rcu);
++}
++EXPORT_SYMBOL_GPL(devl_dpipe_table_unregister);
++
++/**
++ * devl_resource_register - devlink resource register
++ *
++ * @devlink: devlink
++ * @resource_name: resource's name
++ * @resource_size: resource's size
++ * @resource_id: resource's id
++ * @parent_resource_id: resource's parent id
++ * @size_params: size parameters
++ *
++ * Generic resources should reuse the same names across drivers.
++ * Please see the generic resources list at:
++ * Documentation/networking/devlink/devlink-resource.rst
++ */
++int devl_resource_register(struct devlink *devlink,
++			   const char *resource_name,
++			   u64 resource_size,
++			   u64 resource_id,
++			   u64 parent_resource_id,
++			   const struct devlink_resource_size_params *size_params)
++{
++	struct devlink_resource *resource;
++	struct list_head *resource_list;
++	bool top_hierarchy;
++
++	lockdep_assert_held(&devlink->lock);
++
++	top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
++
++	resource = devlink_resource_find(devlink, NULL, resource_id);
++	if (resource)
++		return -EINVAL;
++
++	resource = kzalloc(sizeof(*resource), GFP_KERNEL);
++	if (!resource)
++		return -ENOMEM;
++
++	if (top_hierarchy) {
++		resource_list = &devlink->resource_list;
++	} else {
++		struct devlink_resource *parent_resource;
++
++		parent_resource = devlink_resource_find(devlink, NULL,
++							parent_resource_id);
++		if (parent_resource) {
++			resource_list = &parent_resource->resource_list;
++			resource->parent = parent_resource;
++		} else {
++			kfree(resource);
++			return -EINVAL;
++		}
++	}
++
++	resource->name = resource_name;
++	resource->size = resource_size;
++	resource->size_new = resource_size;
++	resource->id = resource_id;
++	resource->size_valid = true;
++	memcpy(&resource->size_params, size_params,
++	       sizeof(resource->size_params));
++	INIT_LIST_HEAD(&resource->resource_list);
++	list_add_tail(&resource->list, resource_list);
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_resource_register);
++
++/**
++ *	devlink_resource_register - devlink resource register
++ *
++ *	@devlink: devlink
++ *	@resource_name: resource's name
++ *	@resource_size: resource's size
++ *	@resource_id: resource's id
++ *	@parent_resource_id: resource's parent id
++ *	@size_params: size parameters
++ *
++ *	Generic resources should reuse the same names across drivers.
++ *	Please see the generic resources list at:
++ *	Documentation/networking/devlink/devlink-resource.rst
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++int devlink_resource_register(struct devlink *devlink,
++			      const char *resource_name,
++			      u64 resource_size,
++			      u64 resource_id,
++			      u64 parent_resource_id,
++			      const struct devlink_resource_size_params *size_params)
++{
++	int err;
++
++	devl_lock(devlink);
++	err = devl_resource_register(devlink, resource_name, resource_size,
++				     resource_id, parent_resource_id, size_params);
++	devl_unlock(devlink);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_resource_register);
++
++static void devlink_resource_unregister(struct devlink *devlink,
++					struct devlink_resource *resource)
++{
++	struct devlink_resource *tmp, *child_resource;
++
++	list_for_each_entry_safe(child_resource, tmp, &resource->resource_list,
++				 list) {
++		devlink_resource_unregister(devlink, child_resource);
++		list_del(&child_resource->list);
++		kfree(child_resource);
++	}
++}
++
++/**
++ * devl_resources_unregister - free all resources
++ *
++ * @devlink: devlink
++ */
++void devl_resources_unregister(struct devlink *devlink)
++{
++	struct devlink_resource *tmp, *child_resource;
++
++	lockdep_assert_held(&devlink->lock);
++
++	list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list,
++				 list) {
++		devlink_resource_unregister(devlink, child_resource);
++		list_del(&child_resource->list);
++		kfree(child_resource);
++	}
++}
++EXPORT_SYMBOL_GPL(devl_resources_unregister);
++
++/**
++ *	devlink_resources_unregister - free all resources
++ *
++ *	@devlink: devlink
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_resources_unregister(struct devlink *devlink)
++{
++	devl_lock(devlink);
++	devl_resources_unregister(devlink);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_resources_unregister);
++
++/**
++ * devl_resource_size_get - get and update size
++ *
++ * @devlink: devlink
++ * @resource_id: the requested resource id
++ * @p_resource_size: ptr to update
++ */
++int devl_resource_size_get(struct devlink *devlink,
++			   u64 resource_id,
++			   u64 *p_resource_size)
++{
++	struct devlink_resource *resource;
++
++	lockdep_assert_held(&devlink->lock);
++
++	resource = devlink_resource_find(devlink, NULL, resource_id);
++	if (!resource)
++		return -EINVAL;
++	*p_resource_size = resource->size_new;
++	resource->size = resource->size_new;
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_resource_size_get);
++
++/**
++ * devl_dpipe_table_resource_set - set the resource id
++ *
++ * @devlink: devlink
++ * @table_name: table name
++ * @resource_id: resource id
++ * @resource_units: number of resource's units consumed per table's entry
++ */
++int devl_dpipe_table_resource_set(struct devlink *devlink,
++				  const char *table_name, u64 resource_id,
++				  u64 resource_units)
++{
++	struct devlink_dpipe_table *table;
++
++	table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
++					 table_name, devlink);
++	if (!table)
++		return -EINVAL;
++
++	table->resource_id = resource_id;
++	table->resource_units = resource_units;
++	table->resource_valid = true;
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devl_dpipe_table_resource_set);
++
++/**
++ * devl_resource_occ_get_register - register occupancy getter
++ *
++ * @devlink: devlink
++ * @resource_id: resource id
++ * @occ_get: occupancy getter callback
++ * @occ_get_priv: occupancy getter callback priv
++ */
++void devl_resource_occ_get_register(struct devlink *devlink,
++				    u64 resource_id,
++				    devlink_resource_occ_get_t *occ_get,
++				    void *occ_get_priv)
++{
++	struct devlink_resource *resource;
++
++	lockdep_assert_held(&devlink->lock);
++
++	resource = devlink_resource_find(devlink, NULL, resource_id);
++	if (WARN_ON(!resource))
++		return;
++	WARN_ON(resource->occ_get);
++
++	resource->occ_get = occ_get;
++	resource->occ_get_priv = occ_get_priv;
++}
++EXPORT_SYMBOL_GPL(devl_resource_occ_get_register);
++
++/**
++ *	devlink_resource_occ_get_register - register occupancy getter
++ *
++ *	@devlink: devlink
++ *	@resource_id: resource id
++ *	@occ_get: occupancy getter callback
++ *	@occ_get_priv: occupancy getter callback priv
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_resource_occ_get_register(struct devlink *devlink,
++				       u64 resource_id,
++				       devlink_resource_occ_get_t *occ_get,
++				       void *occ_get_priv)
++{
++	devl_lock(devlink);
++	devl_resource_occ_get_register(devlink, resource_id,
++				       occ_get, occ_get_priv);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
++
++/**
++ * devl_resource_occ_get_unregister - unregister occupancy getter
++ *
++ * @devlink: devlink
++ * @resource_id: resource id
++ */
++void devl_resource_occ_get_unregister(struct devlink *devlink,
++				      u64 resource_id)
++{
++	struct devlink_resource *resource;
++
++	lockdep_assert_held(&devlink->lock);
++
++	resource = devlink_resource_find(devlink, NULL, resource_id);
++	if (WARN_ON(!resource))
++		return;
++	WARN_ON(!resource->occ_get);
++
++	resource->occ_get = NULL;
++	resource->occ_get_priv = NULL;
++}
++EXPORT_SYMBOL_GPL(devl_resource_occ_get_unregister);
++
++/**
++ *	devlink_resource_occ_get_unregister - unregister occupancy getter
++ *
++ *	@devlink: devlink
++ *	@resource_id: resource id
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_resource_occ_get_unregister(struct devlink *devlink,
++					 u64 resource_id)
++{
++	devl_lock(devlink);
++	devl_resource_occ_get_unregister(devlink, resource_id);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
++
++static int devlink_param_verify(const struct devlink_param *param)
++{
++	if (!param || !param->name || !param->supported_cmodes)
++		return -EINVAL;
++	if (param->generic)
++		return devlink_param_generic_verify(param);
++	else
++		return devlink_param_driver_verify(param);
++}
++
++/**
++ *	devlink_params_register - register configuration parameters
++ *
++ *	@devlink: devlink
++ *	@params: configuration parameters array
++ *	@params_count: number of parameters provided
++ *
++ *	Register the configuration parameters supported by the driver.
++ */
++int devlink_params_register(struct devlink *devlink,
++			    const struct devlink_param *params,
++			    size_t params_count)
++{
++	const struct devlink_param *param = params;
++	int i, err;
++
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	for (i = 0; i < params_count; i++, param++) {
++		err = devlink_param_register(devlink, param);
++		if (err)
++			goto rollback;
++	}
++	return 0;
++
++rollback:
++	if (!i)
++		return err;
++
++	for (param--; i > 0; i--, param--)
++		devlink_param_unregister(devlink, param);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_params_register);
++
++/**
++ *	devlink_params_unregister - unregister configuration parameters
++ *	@devlink: devlink
++ *	@params: configuration parameters to unregister
++ *	@params_count: number of parameters provided
++ */
++void devlink_params_unregister(struct devlink *devlink,
++			       const struct devlink_param *params,
++			       size_t params_count)
++{
++	const struct devlink_param *param = params;
++	int i;
++
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	for (i = 0; i < params_count; i++, param++)
++		devlink_param_unregister(devlink, param);
++}
++EXPORT_SYMBOL_GPL(devlink_params_unregister);
++
++/**
++ * devlink_param_register - register one configuration parameter
++ *
++ * @devlink: devlink
++ * @param: one configuration parameter
++ *
++ * Register the configuration parameter supported by the driver.
++ * Return: returns 0 on successful registration or error code otherwise.
++ */
++int devlink_param_register(struct devlink *devlink,
++			   const struct devlink_param *param)
++{
++	struct devlink_param_item *param_item;
++
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	WARN_ON(devlink_param_verify(param));
++	WARN_ON(devlink_param_find_by_name(&devlink->param_list, param->name));
++
++	if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
++		WARN_ON(param->get || param->set);
++	else
++		WARN_ON(!param->get || !param->set);
++
++	param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
++	if (!param_item)
++		return -ENOMEM;
++
++	param_item->param = param;
++
++	list_add_tail(&param_item->list, &devlink->param_list);
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_param_register);
++
++/**
++ * devlink_param_unregister - unregister one configuration parameter
++ * @devlink: devlink
++ * @param: configuration parameter to unregister
++ */
++void devlink_param_unregister(struct devlink *devlink,
++			      const struct devlink_param *param)
++{
++	struct devlink_param_item *param_item;
++
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	param_item =
++		devlink_param_find_by_name(&devlink->param_list, param->name);
++	WARN_ON(!param_item);
++	list_del(&param_item->list);
++	kfree(param_item);
++}
++EXPORT_SYMBOL_GPL(devlink_param_unregister);
++
++/**
++ *	devlink_param_driverinit_value_get - get configuration parameter
++ *					     value for driver initializing
++ *
++ *	@devlink: devlink
++ *	@param_id: parameter ID
++ *	@init_val: value of parameter in driverinit configuration mode
++ *
++ *	This function should be used by the driver to get driverinit
++ *	configuration for initialization after reload command.
++ */
++int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
++				       union devlink_param_value *init_val)
++{
++	struct devlink_param_item *param_item;
++
++	if (!devlink_reload_supported(devlink->ops))
++		return -EOPNOTSUPP;
++
++	param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
++	if (!param_item)
++		return -EINVAL;
++
++	if (!param_item->driverinit_value_valid ||
++	    !devlink_param_cmode_is_supported(param_item->param,
++					      DEVLINK_PARAM_CMODE_DRIVERINIT))
++		return -EOPNOTSUPP;
++
++	if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
++		strcpy(init_val->vstr, param_item->driverinit_value.vstr);
++	else
++		*init_val = param_item->driverinit_value;
++
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
++
++/**
++ *	devlink_param_driverinit_value_set - set value of configuration
++ *					     parameter for driverinit
++ *					     configuration mode
++ *
++ *	@devlink: devlink
++ *	@param_id: parameter ID
++ *	@init_val: value of parameter to set for driverinit configuration mode
++ *
++ *	This function should be used by the driver to set driverinit
++ *	configuration mode default value.
++ */
++int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
++				       union devlink_param_value init_val)
++{
++	struct devlink_param_item *param_item;
++
++	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
++
++	param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
++	if (!param_item)
++		return -EINVAL;
++
++	if (!devlink_param_cmode_is_supported(param_item->param,
++					      DEVLINK_PARAM_CMODE_DRIVERINIT))
++		return -EOPNOTSUPP;
++
++	if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
++		strcpy(param_item->driverinit_value.vstr, init_val.vstr);
++	else
++		param_item->driverinit_value = init_val;
++	param_item->driverinit_value_valid = true;
++	return 0;
++}
++EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
++
++/**
++ *	devlink_param_value_changed - notify devlink on a parameter's value
++ *				      change. Should be called by the driver
++ *				      right after the change.
++ *
++ *	@devlink: devlink
++ *	@param_id: parameter ID
++ *
++ *	This function should be used by the driver to notify devlink on value
++ *	change, excluding driverinit configuration mode.
++ *	For driverinit configuration mode driver should use the function
++ */
++void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
++{
++	struct devlink_param_item *param_item;
++
++	param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
++	WARN_ON(!param_item);
++
++	devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
++}
++EXPORT_SYMBOL_GPL(devlink_param_value_changed);
++
++/**
++ * devl_region_create - create a new address region
++ *
++ * @devlink: devlink
++ * @ops: region operations and name
++ * @region_max_snapshots: Maximum supported number of snapshots for region
++ * @region_size: size of region
++ */
++struct devlink_region *devl_region_create(struct devlink *devlink,
++					  const struct devlink_region_ops *ops,
++					  u32 region_max_snapshots,
++					  u64 region_size)
++{
++	struct devlink_region *region;
++
++	devl_assert_locked(devlink);
++
++	if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
++		return ERR_PTR(-EINVAL);
++
++	if (devlink_region_get_by_name(devlink, ops->name))
++		return ERR_PTR(-EEXIST);
++
++	region = kzalloc(sizeof(*region), GFP_KERNEL);
++	if (!region)
++		return ERR_PTR(-ENOMEM);
++
++	region->devlink = devlink;
++	region->max_snapshots = region_max_snapshots;
++	region->ops = ops;
++	region->size = region_size;
++	INIT_LIST_HEAD(&region->snapshot_list);
++	mutex_init(&region->snapshot_lock);
++	list_add_tail(&region->list, &devlink->region_list);
++	devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
++
++	return region;
++}
++EXPORT_SYMBOL_GPL(devl_region_create);
++
++/**
++ *	devlink_region_create - create a new address region
++ *
++ *	@devlink: devlink
++ *	@ops: region operations and name
++ *	@region_max_snapshots: Maximum supported number of snapshots for region
++ *	@region_size: size of region
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++struct devlink_region *
++devlink_region_create(struct devlink *devlink,
++		      const struct devlink_region_ops *ops,
++		      u32 region_max_snapshots, u64 region_size)
++{
++	struct devlink_region *region;
++
++	devl_lock(devlink);
++	region = devl_region_create(devlink, ops, region_max_snapshots,
++				    region_size);
++	devl_unlock(devlink);
++	return region;
++}
++EXPORT_SYMBOL_GPL(devlink_region_create);
++
++/**
++ *	devlink_port_region_create - create a new address region for a port
++ *
++ *	@port: devlink port
++ *	@ops: region operations and name
++ *	@region_max_snapshots: Maximum supported number of snapshots for region
++ *	@region_size: size of region
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++struct devlink_region *
++devlink_port_region_create(struct devlink_port *port,
++			   const struct devlink_port_region_ops *ops,
++			   u32 region_max_snapshots, u64 region_size)
++{
++	struct devlink *devlink = port->devlink;
++	struct devlink_region *region;
++	int err = 0;
++
++	ASSERT_DEVLINK_PORT_INITIALIZED(port);
++
++	if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
++		return ERR_PTR(-EINVAL);
++
++	devl_lock(devlink);
++
++	if (devlink_port_region_get_by_name(port, ops->name)) {
++		err = -EEXIST;
++		goto unlock;
++	}
++
++	region = kzalloc(sizeof(*region), GFP_KERNEL);
++	if (!region) {
++		err = -ENOMEM;
++		goto unlock;
++	}
++
++	region->devlink = devlink;
++	region->port = port;
++	region->max_snapshots = region_max_snapshots;
++	region->port_ops = ops;
++	region->size = region_size;
++	INIT_LIST_HEAD(&region->snapshot_list);
++	mutex_init(&region->snapshot_lock);
++	list_add_tail(&region->list, &port->region_list);
++	devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
++
++	devl_unlock(devlink);
++	return region;
++
++unlock:
++	devl_unlock(devlink);
++	return ERR_PTR(err);
++}
++EXPORT_SYMBOL_GPL(devlink_port_region_create);
++
++/**
++ * devl_region_destroy - destroy address region
++ *
++ * @region: devlink region to destroy
++ */
++void devl_region_destroy(struct devlink_region *region)
++{
++	struct devlink *devlink = region->devlink;
++	struct devlink_snapshot *snapshot, *ts;
++
++	devl_assert_locked(devlink);
++
++	/* Free all snapshots of region */
++	mutex_lock(&region->snapshot_lock);
++	list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
++		devlink_region_snapshot_del(region, snapshot);
++	mutex_unlock(&region->snapshot_lock);
++
++	list_del(&region->list);
++	mutex_destroy(&region->snapshot_lock);
++
++	devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
++	kfree(region);
++}
++EXPORT_SYMBOL_GPL(devl_region_destroy);
++
++/**
++ *	devlink_region_destroy - destroy address region
++ *
++ *	@region: devlink region to destroy
++ *
++ *	Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_region_destroy(struct devlink_region *region)
++{
++	struct devlink *devlink = region->devlink;
++
++	devl_lock(devlink);
++	devl_region_destroy(region);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_region_destroy);
++
++/**
++ *	devlink_region_snapshot_id_get - get snapshot ID
++ *
++ *	This callback should be called when adding a new snapshot,
++ *	Driver should use the same id for multiple snapshots taken
++ *	on multiple regions at the same time/by the same trigger.
++ *
++ *	The caller of this function must use devlink_region_snapshot_id_put
++ *	when finished creating regions using this id.
++ *
++ *	Returns zero on success, or a negative error code on failure.
++ *
++ *	@devlink: devlink
++ *	@id: storage to return id
++ */
++int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
++{
++	return __devlink_region_snapshot_id_get(devlink, id);
++}
++EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
++
++/**
++ *	devlink_region_snapshot_id_put - put snapshot ID reference
++ *
++ *	This should be called by a driver after finishing creating snapshots
++ *	with an id. Doing so ensures that the ID can later be released in the
++ *	event that all snapshots using it have been destroyed.
++ *
++ *	@devlink: devlink
++ *	@id: id to release reference on
++ */
++void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
++{
++	__devlink_snapshot_id_decrement(devlink, id);
++}
++EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
++
++/**
++ *	devlink_region_snapshot_create - create a new snapshot
++ *	This will add a new snapshot of a region. The snapshot
++ *	will be stored on the region struct and can be accessed
++ *	from devlink. This is useful for future analyses of snapshots.
++ *	Multiple snapshots can be created on a region.
++ *	The @snapshot_id should be obtained using the getter function.
++ *
++ *	@region: devlink region of the snapshot
++ *	@data: snapshot data
++ *	@snapshot_id: snapshot id to be created
++ */
++int devlink_region_snapshot_create(struct devlink_region *region,
++				   u8 *data, u32 snapshot_id)
++{
++	int err;
++
++	mutex_lock(&region->snapshot_lock);
++	err = __devlink_region_snapshot_create(region, data, snapshot_id);
++	mutex_unlock(&region->snapshot_lock);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
++
++#define DEVLINK_TRAP(_id, _type)					      \
++	{								      \
++		.type = DEVLINK_TRAP_TYPE_##_type,			      \
++		.id = DEVLINK_TRAP_GENERIC_ID_##_id,			      \
++		.name = DEVLINK_TRAP_GENERIC_NAME_##_id,		      \
++	}
++
++static const struct devlink_trap devlink_trap_generic[] = {
++	DEVLINK_TRAP(SMAC_MC, DROP),
++	DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
++	DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
++	DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
++	DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
++	DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
++	DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
++	DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
++	DEVLINK_TRAP(TAIL_DROP, DROP),
++	DEVLINK_TRAP(NON_IP_PACKET, DROP),
++	DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
++	DEVLINK_TRAP(DIP_LB, DROP),
++	DEVLINK_TRAP(SIP_MC, DROP),
++	DEVLINK_TRAP(SIP_LB, DROP),
++	DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
++	DEVLINK_TRAP(IPV4_SIP_BC, DROP),
++	DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
++	DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
++	DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
++	DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
++	DEVLINK_TRAP(RPF, EXCEPTION),
++	DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
++	DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
++	DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
++	DEVLINK_TRAP(NON_ROUTABLE, DROP),
++	DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
++	DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
++	DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
++	DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
++	DEVLINK_TRAP(STP, CONTROL),
++	DEVLINK_TRAP(LACP, CONTROL),
++	DEVLINK_TRAP(LLDP, CONTROL),
++	DEVLINK_TRAP(IGMP_QUERY, CONTROL),
++	DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
++	DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
++	DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
++	DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
++	DEVLINK_TRAP(MLD_QUERY, CONTROL),
++	DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
++	DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
++	DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
++	DEVLINK_TRAP(IPV4_DHCP, CONTROL),
++	DEVLINK_TRAP(IPV6_DHCP, CONTROL),
++	DEVLINK_TRAP(ARP_REQUEST, CONTROL),
++	DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
++	DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
++	DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
++	DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
++	DEVLINK_TRAP(IPV4_BFD, CONTROL),
++	DEVLINK_TRAP(IPV6_BFD, CONTROL),
++	DEVLINK_TRAP(IPV4_OSPF, CONTROL),
++	DEVLINK_TRAP(IPV6_OSPF, CONTROL),
++	DEVLINK_TRAP(IPV4_BGP, CONTROL),
++	DEVLINK_TRAP(IPV6_BGP, CONTROL),
++	DEVLINK_TRAP(IPV4_VRRP, CONTROL),
++	DEVLINK_TRAP(IPV6_VRRP, CONTROL),
++	DEVLINK_TRAP(IPV4_PIM, CONTROL),
++	DEVLINK_TRAP(IPV6_PIM, CONTROL),
++	DEVLINK_TRAP(UC_LB, CONTROL),
++	DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
++	DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
++	DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
++	DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
++	DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
++	DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
++	DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
++	DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
++	DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
++	DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
++	DEVLINK_TRAP(PTP_EVENT, CONTROL),
++	DEVLINK_TRAP(PTP_GENERAL, CONTROL),
++	DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
++	DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
++	DEVLINK_TRAP(EARLY_DROP, DROP),
++	DEVLINK_TRAP(VXLAN_PARSING, DROP),
++	DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
++	DEVLINK_TRAP(VLAN_PARSING, DROP),
++	DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
++	DEVLINK_TRAP(MPLS_PARSING, DROP),
++	DEVLINK_TRAP(ARP_PARSING, DROP),
++	DEVLINK_TRAP(IP_1_PARSING, DROP),
++	DEVLINK_TRAP(IP_N_PARSING, DROP),
++	DEVLINK_TRAP(GRE_PARSING, DROP),
++	DEVLINK_TRAP(UDP_PARSING, DROP),
++	DEVLINK_TRAP(TCP_PARSING, DROP),
++	DEVLINK_TRAP(IPSEC_PARSING, DROP),
++	DEVLINK_TRAP(SCTP_PARSING, DROP),
++	DEVLINK_TRAP(DCCP_PARSING, DROP),
++	DEVLINK_TRAP(GTP_PARSING, DROP),
++	DEVLINK_TRAP(ESP_PARSING, DROP),
++	DEVLINK_TRAP(BLACKHOLE_NEXTHOP, DROP),
++	DEVLINK_TRAP(DMAC_FILTER, DROP),
++};
++
++#define DEVLINK_TRAP_GROUP(_id)						      \
++	{								      \
++		.id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,		      \
++		.name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,		      \
++	}
++
++static const struct devlink_trap_group devlink_trap_group_generic[] = {
++	DEVLINK_TRAP_GROUP(L2_DROPS),
++	DEVLINK_TRAP_GROUP(L3_DROPS),
++	DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
++	DEVLINK_TRAP_GROUP(BUFFER_DROPS),
++	DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
++	DEVLINK_TRAP_GROUP(ACL_DROPS),
++	DEVLINK_TRAP_GROUP(STP),
++	DEVLINK_TRAP_GROUP(LACP),
++	DEVLINK_TRAP_GROUP(LLDP),
++	DEVLINK_TRAP_GROUP(MC_SNOOPING),
++	DEVLINK_TRAP_GROUP(DHCP),
++	DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
++	DEVLINK_TRAP_GROUP(BFD),
++	DEVLINK_TRAP_GROUP(OSPF),
++	DEVLINK_TRAP_GROUP(BGP),
++	DEVLINK_TRAP_GROUP(VRRP),
++	DEVLINK_TRAP_GROUP(PIM),
++	DEVLINK_TRAP_GROUP(UC_LB),
++	DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
++	DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
++	DEVLINK_TRAP_GROUP(IPV6),
++	DEVLINK_TRAP_GROUP(PTP_EVENT),
++	DEVLINK_TRAP_GROUP(PTP_GENERAL),
++	DEVLINK_TRAP_GROUP(ACL_SAMPLE),
++	DEVLINK_TRAP_GROUP(ACL_TRAP),
++	DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
++};
++
++static int devlink_trap_generic_verify(const struct devlink_trap *trap)
++{
++	if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
++		return -EINVAL;
++
++	if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
++		return -EINVAL;
++
++	if (trap->type != devlink_trap_generic[trap->id].type)
++		return -EINVAL;
++
++	return 0;
++}
++
++static int devlink_trap_driver_verify(const struct devlink_trap *trap)
++{
++	int i;
++
++	if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
++		return -EINVAL;
++
++	for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
++		if (!strcmp(trap->name, devlink_trap_generic[i].name))
++			return -EEXIST;
++	}
++
++	return 0;
++}
++
++static int devlink_trap_verify(const struct devlink_trap *trap)
++{
++	if (!trap || !trap->name)
++		return -EINVAL;
++
++	if (trap->generic)
++		return devlink_trap_generic_verify(trap);
++	else
++		return devlink_trap_driver_verify(trap);
++}
++
++static int
++devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
++{
++	if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
++		return -EINVAL;
++
++	if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
++		return -EINVAL;
++
++	return 0;
++}
++
++static int
++devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
++{
++	int i;
++
++	if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
++		return -EINVAL;
++
++	for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
++		if (!strcmp(group->name, devlink_trap_group_generic[i].name))
++			return -EEXIST;
++	}
++
++	return 0;
++}
++
++static int devlink_trap_group_verify(const struct devlink_trap_group *group)
++{
++	if (group->generic)
++		return devlink_trap_group_generic_verify(group);
++	else
++		return devlink_trap_group_driver_verify(group);
++}
++
++static void
++devlink_trap_group_notify(struct devlink *devlink,
++			  const struct devlink_trap_group_item *group_item,
++			  enum devlink_command cmd)
++{
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
++		     cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
++					 0);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int
++devlink_trap_item_group_link(struct devlink *devlink,
++			     struct devlink_trap_item *trap_item)
++{
++	u16 group_id = trap_item->trap->init_group_id;
++	struct devlink_trap_group_item *group_item;
++
++	group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
++	if (WARN_ON_ONCE(!group_item))
++		return -EINVAL;
++
++	trap_item->group_item = group_item;
++
++	return 0;
++}
++
++static void devlink_trap_notify(struct devlink *devlink,
++				const struct devlink_trap_item *trap_item,
++				enum devlink_command cmd)
++{
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
++		     cmd != DEVLINK_CMD_TRAP_DEL);
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int
++devlink_trap_register(struct devlink *devlink,
++		      const struct devlink_trap *trap, void *priv)
++{
++	struct devlink_trap_item *trap_item;
++	int err;
++
++	if (devlink_trap_item_lookup(devlink, trap->name))
++		return -EEXIST;
++
++	trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
++	if (!trap_item)
++		return -ENOMEM;
++
++	trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
++	if (!trap_item->stats) {
++		err = -ENOMEM;
++		goto err_stats_alloc;
++	}
++
++	trap_item->trap = trap;
++	trap_item->action = trap->init_action;
++	trap_item->priv = priv;
++
++	err = devlink_trap_item_group_link(devlink, trap_item);
++	if (err)
++		goto err_group_link;
++
++	err = devlink->ops->trap_init(devlink, trap, trap_item);
++	if (err)
++		goto err_trap_init;
++
++	list_add_tail(&trap_item->list, &devlink->trap_list);
++	devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
++
++	return 0;
++
++err_trap_init:
++err_group_link:
++	free_percpu(trap_item->stats);
++err_stats_alloc:
++	kfree(trap_item);
++	return err;
++}
++
++static void devlink_trap_unregister(struct devlink *devlink,
++				    const struct devlink_trap *trap)
++{
++	struct devlink_trap_item *trap_item;
++
++	trap_item = devlink_trap_item_lookup(devlink, trap->name);
++	if (WARN_ON_ONCE(!trap_item))
++		return;
++
++	devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
++	list_del(&trap_item->list);
++	if (devlink->ops->trap_fini)
++		devlink->ops->trap_fini(devlink, trap, trap_item);
++	free_percpu(trap_item->stats);
++	kfree(trap_item);
++}
++
++static void devlink_trap_disable(struct devlink *devlink,
++				 const struct devlink_trap *trap)
++{
++	struct devlink_trap_item *trap_item;
++
++	trap_item = devlink_trap_item_lookup(devlink, trap->name);
++	if (WARN_ON_ONCE(!trap_item))
++		return;
++
++	devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
++				      NULL);
++	trap_item->action = DEVLINK_TRAP_ACTION_DROP;
++}
++
++/**
++ * devl_traps_register - Register packet traps with devlink.
++ * @devlink: devlink.
++ * @traps: Packet traps.
++ * @traps_count: Count of provided packet traps.
++ * @priv: Driver private information.
++ *
++ * Return: Non-zero value on failure.
++ */
++int devl_traps_register(struct devlink *devlink,
++			const struct devlink_trap *traps,
++			size_t traps_count, void *priv)
++{
++	int i, err;
++
++	if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
++		return -EINVAL;
++
++	devl_assert_locked(devlink);
++	for (i = 0; i < traps_count; i++) {
++		const struct devlink_trap *trap = &traps[i];
++
++		err = devlink_trap_verify(trap);
++		if (err)
++			goto err_trap_verify;
++
++		err = devlink_trap_register(devlink, trap, priv);
++		if (err)
++			goto err_trap_register;
++	}
++
++	return 0;
++
++err_trap_register:
++err_trap_verify:
++	for (i--; i >= 0; i--)
++		devlink_trap_unregister(devlink, &traps[i]);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devl_traps_register);
++
++/**
++ * devlink_traps_register - Register packet traps with devlink.
++ * @devlink: devlink.
++ * @traps: Packet traps.
++ * @traps_count: Count of provided packet traps.
++ * @priv: Driver private information.
++ *
++ * Context: Takes and release devlink->lock <mutex>.
++ *
++ * Return: Non-zero value on failure.
++ */
++int devlink_traps_register(struct devlink *devlink,
++			   const struct devlink_trap *traps,
++			   size_t traps_count, void *priv)
++{
++	int err;
++
++	devl_lock(devlink);
++	err = devl_traps_register(devlink, traps, traps_count, priv);
++	devl_unlock(devlink);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_traps_register);
++
++/**
++ * devl_traps_unregister - Unregister packet traps from devlink.
++ * @devlink: devlink.
++ * @traps: Packet traps.
++ * @traps_count: Count of provided packet traps.
++ */
++void devl_traps_unregister(struct devlink *devlink,
++			   const struct devlink_trap *traps,
++			   size_t traps_count)
++{
++	int i;
++
++	devl_assert_locked(devlink);
++	/* Make sure we do not have any packets in-flight while unregistering
++	 * traps by disabling all of them and waiting for a grace period.
++	 */
++	for (i = traps_count - 1; i >= 0; i--)
++		devlink_trap_disable(devlink, &traps[i]);
++	synchronize_rcu();
++	for (i = traps_count - 1; i >= 0; i--)
++		devlink_trap_unregister(devlink, &traps[i]);
++}
++EXPORT_SYMBOL_GPL(devl_traps_unregister);
++
++/**
++ * devlink_traps_unregister - Unregister packet traps from devlink.
++ * @devlink: devlink.
++ * @traps: Packet traps.
++ * @traps_count: Count of provided packet traps.
++ *
++ * Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_traps_unregister(struct devlink *devlink,
++			      const struct devlink_trap *traps,
++			      size_t traps_count)
++{
++	devl_lock(devlink);
++	devl_traps_unregister(devlink, traps, traps_count);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_traps_unregister);
++
++static void
++devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
++			  size_t skb_len)
++{
++	struct devlink_stats *stats;
++
++	stats = this_cpu_ptr(trap_stats);
++	u64_stats_update_begin(&stats->syncp);
++	u64_stats_add(&stats->rx_bytes, skb_len);
++	u64_stats_inc(&stats->rx_packets);
++	u64_stats_update_end(&stats->syncp);
++}
++
++static void
++devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
++				 const struct devlink_trap_item *trap_item,
++				 struct devlink_port *in_devlink_port,
++				 const struct flow_action_cookie *fa_cookie)
++{
++	metadata->trap_name = trap_item->trap->name;
++	metadata->trap_group_name = trap_item->group_item->group->name;
++	metadata->fa_cookie = fa_cookie;
++	metadata->trap_type = trap_item->trap->type;
++
++	spin_lock(&in_devlink_port->type_lock);
++	if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
++		metadata->input_dev = in_devlink_port->type_dev;
++	spin_unlock(&in_devlink_port->type_lock);
++}
++
++/**
++ * devlink_trap_report - Report trapped packet to drop monitor.
++ * @devlink: devlink.
++ * @skb: Trapped packet.
++ * @trap_ctx: Trap context.
++ * @in_devlink_port: Input devlink port.
++ * @fa_cookie: Flow action cookie. Could be NULL.
++ */
++void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
++			 void *trap_ctx, struct devlink_port *in_devlink_port,
++			 const struct flow_action_cookie *fa_cookie)
++
++{
++	struct devlink_trap_item *trap_item = trap_ctx;
++
++	devlink_trap_stats_update(trap_item->stats, skb->len);
++	devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
++
++	if (trace_devlink_trap_report_enabled()) {
++		struct devlink_trap_metadata metadata = {};
++
++		devlink_trap_report_metadata_set(&metadata, trap_item,
++						 in_devlink_port, fa_cookie);
++		trace_devlink_trap_report(devlink, skb, &metadata);
++	}
++}
++EXPORT_SYMBOL_GPL(devlink_trap_report);
++
++/**
++ * devlink_trap_ctx_priv - Trap context to driver private information.
++ * @trap_ctx: Trap context.
++ *
++ * Return: Driver private information passed during registration.
++ */
++void *devlink_trap_ctx_priv(void *trap_ctx)
++{
++	struct devlink_trap_item *trap_item = trap_ctx;
++
++	return trap_item->priv;
++}
++EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
++
++static int
++devlink_trap_group_item_policer_link(struct devlink *devlink,
++				     struct devlink_trap_group_item *group_item)
++{
++	u32 policer_id = group_item->group->init_policer_id;
++	struct devlink_trap_policer_item *policer_item;
++
++	if (policer_id == 0)
++		return 0;
++
++	policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
++	if (WARN_ON_ONCE(!policer_item))
++		return -EINVAL;
++
++	group_item->policer_item = policer_item;
++
++	return 0;
++}
++
++static int
++devlink_trap_group_register(struct devlink *devlink,
++			    const struct devlink_trap_group *group)
++{
++	struct devlink_trap_group_item *group_item;
++	int err;
++
++	if (devlink_trap_group_item_lookup(devlink, group->name))
++		return -EEXIST;
++
++	group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
++	if (!group_item)
++		return -ENOMEM;
++
++	group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
++	if (!group_item->stats) {
++		err = -ENOMEM;
++		goto err_stats_alloc;
++	}
++
++	group_item->group = group;
++
++	err = devlink_trap_group_item_policer_link(devlink, group_item);
++	if (err)
++		goto err_policer_link;
++
++	if (devlink->ops->trap_group_init) {
++		err = devlink->ops->trap_group_init(devlink, group);
++		if (err)
++			goto err_group_init;
++	}
++
++	list_add_tail(&group_item->list, &devlink->trap_group_list);
++	devlink_trap_group_notify(devlink, group_item,
++				  DEVLINK_CMD_TRAP_GROUP_NEW);
++
++	return 0;
++
++err_group_init:
++err_policer_link:
++	free_percpu(group_item->stats);
++err_stats_alloc:
++	kfree(group_item);
++	return err;
++}
++
++static void
++devlink_trap_group_unregister(struct devlink *devlink,
++			      const struct devlink_trap_group *group)
++{
++	struct devlink_trap_group_item *group_item;
++
++	group_item = devlink_trap_group_item_lookup(devlink, group->name);
++	if (WARN_ON_ONCE(!group_item))
++		return;
++
++	devlink_trap_group_notify(devlink, group_item,
++				  DEVLINK_CMD_TRAP_GROUP_DEL);
++	list_del(&group_item->list);
++	free_percpu(group_item->stats);
++	kfree(group_item);
++}
++
++/**
++ * devl_trap_groups_register - Register packet trap groups with devlink.
++ * @devlink: devlink.
++ * @groups: Packet trap groups.
++ * @groups_count: Count of provided packet trap groups.
++ *
++ * Return: Non-zero value on failure.
++ */
++int devl_trap_groups_register(struct devlink *devlink,
++			      const struct devlink_trap_group *groups,
++			      size_t groups_count)
++{
++	int i, err;
++
++	devl_assert_locked(devlink);
++	for (i = 0; i < groups_count; i++) {
++		const struct devlink_trap_group *group = &groups[i];
++
++		err = devlink_trap_group_verify(group);
++		if (err)
++			goto err_trap_group_verify;
++
++		err = devlink_trap_group_register(devlink, group);
++		if (err)
++			goto err_trap_group_register;
++	}
++
++	return 0;
++
++err_trap_group_register:
++err_trap_group_verify:
++	for (i--; i >= 0; i--)
++		devlink_trap_group_unregister(devlink, &groups[i]);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devl_trap_groups_register);
++
++/**
++ * devlink_trap_groups_register - Register packet trap groups with devlink.
++ * @devlink: devlink.
++ * @groups: Packet trap groups.
++ * @groups_count: Count of provided packet trap groups.
++ *
++ * Context: Takes and release devlink->lock <mutex>.
++ *
++ * Return: Non-zero value on failure.
++ */
++int devlink_trap_groups_register(struct devlink *devlink,
++				 const struct devlink_trap_group *groups,
++				 size_t groups_count)
++{
++	int err;
++
++	devl_lock(devlink);
++	err = devl_trap_groups_register(devlink, groups, groups_count);
++	devl_unlock(devlink);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
++
++/**
++ * devl_trap_groups_unregister - Unregister packet trap groups from devlink.
++ * @devlink: devlink.
++ * @groups: Packet trap groups.
++ * @groups_count: Count of provided packet trap groups.
++ */
++void devl_trap_groups_unregister(struct devlink *devlink,
++				 const struct devlink_trap_group *groups,
++				 size_t groups_count)
++{
++	int i;
++
++	devl_assert_locked(devlink);
++	for (i = groups_count - 1; i >= 0; i--)
++		devlink_trap_group_unregister(devlink, &groups[i]);
++}
++EXPORT_SYMBOL_GPL(devl_trap_groups_unregister);
++
++/**
++ * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
++ * @devlink: devlink.
++ * @groups: Packet trap groups.
++ * @groups_count: Count of provided packet trap groups.
++ *
++ * Context: Takes and release devlink->lock <mutex>.
++ */
++void devlink_trap_groups_unregister(struct devlink *devlink,
++				    const struct devlink_trap_group *groups,
++				    size_t groups_count)
++{
++	devl_lock(devlink);
++	devl_trap_groups_unregister(devlink, groups, groups_count);
++	devl_unlock(devlink);
++}
++EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
++
++static void
++devlink_trap_policer_notify(struct devlink *devlink,
++			    const struct devlink_trap_policer_item *policer_item,
++			    enum devlink_command cmd)
++{
++	struct sk_buff *msg;
++	int err;
++
++	WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
++		     cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
++	if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
++		return;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
++					   0, 0);
++	if (err) {
++		nlmsg_free(msg);
++		return;
++	}
++
++	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
++				msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
++}
++
++static int
++devlink_trap_policer_register(struct devlink *devlink,
++			      const struct devlink_trap_policer *policer)
++{
++	struct devlink_trap_policer_item *policer_item;
++	int err;
++
++	if (devlink_trap_policer_item_lookup(devlink, policer->id))
++		return -EEXIST;
++
++	policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
++	if (!policer_item)
++		return -ENOMEM;
++
++	policer_item->policer = policer;
++	policer_item->rate = policer->init_rate;
++	policer_item->burst = policer->init_burst;
++
++	if (devlink->ops->trap_policer_init) {
++		err = devlink->ops->trap_policer_init(devlink, policer);
++		if (err)
++			goto err_policer_init;
++	}
++
++	list_add_tail(&policer_item->list, &devlink->trap_policer_list);
++	devlink_trap_policer_notify(devlink, policer_item,
++				    DEVLINK_CMD_TRAP_POLICER_NEW);
++
++	return 0;
++
++err_policer_init:
++	kfree(policer_item);
++	return err;
++}
++
++static void
++devlink_trap_policer_unregister(struct devlink *devlink,
++				const struct devlink_trap_policer *policer)
++{
++	struct devlink_trap_policer_item *policer_item;
++
++	policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
++	if (WARN_ON_ONCE(!policer_item))
++		return;
++
++	devlink_trap_policer_notify(devlink, policer_item,
++				    DEVLINK_CMD_TRAP_POLICER_DEL);
++	list_del(&policer_item->list);
++	if (devlink->ops->trap_policer_fini)
++		devlink->ops->trap_policer_fini(devlink, policer);
++	kfree(policer_item);
++}
++
++/**
++ * devl_trap_policers_register - Register packet trap policers with devlink.
++ * @devlink: devlink.
++ * @policers: Packet trap policers.
++ * @policers_count: Count of provided packet trap policers.
++ *
++ * Return: Non-zero value on failure.
++ */
++int
++devl_trap_policers_register(struct devlink *devlink,
++			    const struct devlink_trap_policer *policers,
++			    size_t policers_count)
++{
++	int i, err;
++
++	devl_assert_locked(devlink);
++	for (i = 0; i < policers_count; i++) {
++		const struct devlink_trap_policer *policer = &policers[i];
++
++		if (WARN_ON(policer->id == 0 ||
++			    policer->max_rate < policer->min_rate ||
++			    policer->max_burst < policer->min_burst)) {
++			err = -EINVAL;
++			goto err_trap_policer_verify;
++		}
++
++		err = devlink_trap_policer_register(devlink, policer);
++		if (err)
++			goto err_trap_policer_register;
++	}
++	return 0;
++
++err_trap_policer_register:
++err_trap_policer_verify:
++	for (i--; i >= 0; i--)
++		devlink_trap_policer_unregister(devlink, &policers[i]);
++	return err;
++}
++EXPORT_SYMBOL_GPL(devl_trap_policers_register);
++
++/**
++ * devl_trap_policers_unregister - Unregister packet trap policers from devlink.
++ * @devlink: devlink.
++ * @policers: Packet trap policers.
++ * @policers_count: Count of provided packet trap policers.
++ */
++void
++devl_trap_policers_unregister(struct devlink *devlink,
++			      const struct devlink_trap_policer *policers,
++			      size_t policers_count)
++{
++	int i;
++
++	devl_assert_locked(devlink);
++	for (i = policers_count - 1; i >= 0; i--)
++		devlink_trap_policer_unregister(devlink, &policers[i]);
++}
++EXPORT_SYMBOL_GPL(devl_trap_policers_unregister);
++
++static void __devlink_compat_running_version(struct devlink *devlink,
++					     char *buf, size_t len)
++{
++	struct devlink_info_req req = {};
++	const struct nlattr *nlattr;
++	struct sk_buff *msg;
++	int rem, err;
++
++	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
++	if (!msg)
++		return;
++
++	req.msg = msg;
++	err = devlink->ops->info_get(devlink, &req, NULL);
++	if (err)
++		goto free_msg;
++
++	nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
++		const struct nlattr *kv;
++		int rem_kv;
++
++		if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
++			continue;
++
++		nla_for_each_nested(kv, nlattr, rem_kv) {
++			if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
++				continue;
++
++			strlcat(buf, nla_data(kv), len);
++			strlcat(buf, " ", len);
++		}
++	}
++free_msg:
++	nlmsg_free(msg);
++}
++
++static struct devlink_port *netdev_to_devlink_port(struct net_device *dev)
++{
++	if (!dev->netdev_ops->ndo_get_devlink_port)
++		return NULL;
++
++	return dev->netdev_ops->ndo_get_devlink_port(dev);
++}
++
++void devlink_compat_running_version(struct devlink *devlink,
++				    char *buf, size_t len)
++{
++	if (!devlink->ops->info_get)
++		return;
++
++	devl_lock(devlink);
++	__devlink_compat_running_version(devlink, buf, len);
++	devl_unlock(devlink);
++}
++
++int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
++{
++	struct devlink_flash_update_params params = {};
++	int ret;
++
++	if (!devlink->ops->flash_update)
++		return -EOPNOTSUPP;
++
++	ret = request_firmware(&params.fw, file_name, devlink->dev);
++	if (ret)
++		return ret;
++
++	devl_lock(devlink);
++	devlink_flash_update_begin_notify(devlink);
++	ret = devlink->ops->flash_update(devlink, &params, NULL);
++	devlink_flash_update_end_notify(devlink);
++	devl_unlock(devlink);
++
++	release_firmware(params.fw);
++
++	return ret;
++}
++
++int devlink_compat_phys_port_name_get(struct net_device *dev,
++				      char *name, size_t len)
++{
++	struct devlink_port *devlink_port;
++
++	/* RTNL mutex is held here which ensures that devlink_port
++	 * instance cannot disappear in the middle. No need to take
++	 * any devlink lock as only permanent values are accessed.
++	 */
++	ASSERT_RTNL();
++
++	devlink_port = netdev_to_devlink_port(dev);
++	if (!devlink_port)
++		return -EOPNOTSUPP;
++
++	return __devlink_port_phys_port_name_get(devlink_port, name, len);
++}
++
++int devlink_compat_switch_id_get(struct net_device *dev,
++				 struct netdev_phys_item_id *ppid)
++{
++	struct devlink_port *devlink_port;
++
++	/* Caller must hold RTNL mutex or reference to dev, which ensures that
++	 * devlink_port instance cannot disappear in the middle. No need to take
++	 * any devlink lock as only permanent values are accessed.
++	 */
++	devlink_port = netdev_to_devlink_port(dev);
++	if (!devlink_port || !devlink_port->switch_port)
++		return -EOPNOTSUPP;
++
++	memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
++
++	return 0;
++}
++
++static void __net_exit devlink_pernet_pre_exit(struct net *net)
++{
++	struct devlink *devlink;
++	u32 actions_performed;
++	unsigned long index;
++	int err;
++
++	/* In case network namespace is getting destroyed, reload
++	 * all devlink instances from this namespace into init_net.
++	 */
++	devlinks_xa_for_each_registered_get(net, index, devlink) {
++		WARN_ON(!(devlink->features & DEVLINK_F_RELOAD));
++		mutex_lock(&devlink->lock);
++		err = devlink_reload(devlink, &init_net,
++				     DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
++				     DEVLINK_RELOAD_LIMIT_UNSPEC,
++				     &actions_performed, NULL);
++		mutex_unlock(&devlink->lock);
++		if (err && err != -EOPNOTSUPP)
++			pr_warn("Failed to reload devlink instance into init_net\n");
++		devlink_put(devlink);
++	}
++}
++
++static struct pernet_operations devlink_pernet_ops __net_initdata = {
++	.pre_exit = devlink_pernet_pre_exit,
++};
++
++static int __init devlink_init(void)
++{
++	int err;
++
++	err = genl_register_family(&devlink_nl_family);
++	if (err)
++		goto out;
++	err = register_pernet_subsys(&devlink_pernet_ops);
++
++out:
++	WARN_ON(err);
++	return err;
++}
++
++subsys_initcall(devlink_init);
+diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
+index ebb737ac9e894..04853c83c85c4 100644
+--- a/net/ipv4/af_inet.c
++++ b/net/ipv4/af_inet.c
+@@ -340,7 +340,7 @@ lookup_protocol:
+ 	else
+ 		inet->pmtudisc = IP_PMTUDISC_WANT;
+ 
+-	inet->inet_id = 0;
++	atomic_set(&inet->inet_id, 0);
+ 
+ 	sock_init_data(sock, sk);
+ 
+diff --git a/net/ipv4/datagram.c b/net/ipv4/datagram.c
+index 4d1af0cd7d99e..cb5dbee9e018f 100644
+--- a/net/ipv4/datagram.c
++++ b/net/ipv4/datagram.c
+@@ -73,7 +73,7 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len
+ 	reuseport_has_conns_set(sk);
+ 	sk->sk_state = TCP_ESTABLISHED;
+ 	sk_set_txhash(sk);
+-	inet->inet_id = get_random_u16();
++	atomic_set(&inet->inet_id, get_random_u16());
+ 
+ 	sk_dst_set(sk, &rt->dst);
+ 	err = 0;
+diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
+index 08921b96f9728..f9b8a4a1d2edc 100644
+--- a/net/ipv4/tcp_ipv4.c
++++ b/net/ipv4/tcp_ipv4.c
+@@ -312,7 +312,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
+ 					     inet->inet_daddr));
+ 	}
+ 
+-	inet->inet_id = get_random_u16();
++	atomic_set(&inet->inet_id, get_random_u16());
+ 
+ 	if (tcp_fastopen_defer_connect(sk, &err))
+ 		return err;
+@@ -1539,7 +1539,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
+ 	inet_csk(newsk)->icsk_ext_hdr_len = 0;
+ 	if (inet_opt)
+ 		inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
+-	newinet->inet_id = get_random_u16();
++	atomic_set(&newinet->inet_id, get_random_u16());
+ 
+ 	/* Set ToS of the new socket based upon the value of incoming SYN.
+ 	 * ECT bits are set later in tcp_init_transfer().
+diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
+index 0f81492da0b46..55dc0610e8633 100644
+--- a/net/mac80211/rx.c
++++ b/net/mac80211/rx.c
+@@ -1102,7 +1102,8 @@ static inline bool ieee80211_rx_reorder_ready(struct tid_ampdu_rx *tid_agg_rx,
+ 	struct sk_buff *tail = skb_peek_tail(frames);
+ 	struct ieee80211_rx_status *status;
+ 
+-	if (tid_agg_rx->reorder_buf_filtered & BIT_ULL(index))
++	if (tid_agg_rx->reorder_buf_filtered &&
++	    tid_agg_rx->reorder_buf_filtered & BIT_ULL(index))
+ 		return true;
+ 
+ 	if (!tail)
+@@ -1143,7 +1144,8 @@ static void ieee80211_release_reorder_frame(struct ieee80211_sub_if_data *sdata,
+ 	}
+ 
+ no_frame:
+-	tid_agg_rx->reorder_buf_filtered &= ~BIT_ULL(index);
++	if (tid_agg_rx->reorder_buf_filtered)
++		tid_agg_rx->reorder_buf_filtered &= ~BIT_ULL(index);
+ 	tid_agg_rx->head_seq_num = ieee80211_sn_inc(tid_agg_rx->head_seq_num);
+ }
+ 
+@@ -4162,6 +4164,7 @@ void ieee80211_mark_rx_ba_filtered_frames(struct ieee80211_sta *pubsta, u8 tid,
+ 					  u16 ssn, u64 filtered,
+ 					  u16 received_mpdus)
+ {
++	struct ieee80211_local *local;
+ 	struct sta_info *sta;
+ 	struct tid_ampdu_rx *tid_agg_rx;
+ 	struct sk_buff_head frames;
+@@ -4179,6 +4182,11 @@ void ieee80211_mark_rx_ba_filtered_frames(struct ieee80211_sta *pubsta, u8 tid,
+ 
+ 	sta = container_of(pubsta, struct sta_info, sta);
+ 
++	local = sta->sdata->local;
++	WARN_ONCE(local->hw.max_rx_aggregation_subframes > 64,
++		  "RX BA marker can't support max_rx_aggregation_subframes %u > 64\n",
++		  local->hw.max_rx_aggregation_subframes);
++
+ 	if (!ieee80211_rx_data_set_sta(&rx, sta, -1))
+ 		return;
+ 
+diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
+index 4c2df7af73f76..3c5cac9bd9b70 100644
+--- a/net/netfilter/nf_tables_api.c
++++ b/net/netfilter/nf_tables_api.c
+@@ -10509,7 +10509,7 @@ static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
+ 	deleted = 0;
+ 	mutex_lock(&nft_net->commit_mutex);
+ 	if (!list_empty(&nf_tables_destroy_list))
+-		rcu_barrier();
++		nf_tables_trans_destroy_flush_work();
+ again:
+ 	list_for_each_entry(table, &nft_net->tables, list) {
+ 		if (nft_table_has_owner(table) &&
+diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c
+index 32cfd0a84b0e2..8c16681884b7e 100644
+--- a/net/netfilter/nft_set_pipapo.c
++++ b/net/netfilter/nft_set_pipapo.c
+@@ -901,12 +901,14 @@ static void pipapo_lt_bits_adjust(struct nft_pipapo_field *f)
+ static int pipapo_insert(struct nft_pipapo_field *f, const uint8_t *k,
+ 			 int mask_bits)
+ {
+-	int rule = f->rules++, group, ret, bit_offset = 0;
++	int rule = f->rules, group, ret, bit_offset = 0;
+ 
+-	ret = pipapo_resize(f, f->rules - 1, f->rules);
++	ret = pipapo_resize(f, f->rules, f->rules + 1);
+ 	if (ret)
+ 		return ret;
+ 
++	f->rules++;
++
+ 	for (group = 0; group < f->groups; group++) {
+ 		int i, v;
+ 		u8 mask;
+@@ -1051,7 +1053,9 @@ static int pipapo_expand(struct nft_pipapo_field *f,
+ 			step++;
+ 			if (step >= len) {
+ 				if (!masks) {
+-					pipapo_insert(f, base, 0);
++					err = pipapo_insert(f, base, 0);
++					if (err < 0)
++						return err;
+ 					masks = 1;
+ 				}
+ 				goto out;
+@@ -1234,6 +1238,9 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set,
+ 		else
+ 			ret = pipapo_expand(f, start, end, f->groups * f->bb);
+ 
++		if (ret < 0)
++			return ret;
++
+ 		if (f->bsize > bsize_max)
+ 			bsize_max = f->bsize;
+ 
+diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
+index 01d07e6a68119..e8f988e1c7e64 100644
+--- a/net/sched/sch_api.c
++++ b/net/sched/sch_api.c
+@@ -1550,10 +1550,28 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
+ 	return 0;
+ }
+ 
++static bool req_create_or_replace(struct nlmsghdr *n)
++{
++	return (n->nlmsg_flags & NLM_F_CREATE &&
++		n->nlmsg_flags & NLM_F_REPLACE);
++}
++
++static bool req_create_exclusive(struct nlmsghdr *n)
++{
++	return (n->nlmsg_flags & NLM_F_CREATE &&
++		n->nlmsg_flags & NLM_F_EXCL);
++}
++
++static bool req_change(struct nlmsghdr *n)
++{
++	return (!(n->nlmsg_flags & NLM_F_CREATE) &&
++		!(n->nlmsg_flags & NLM_F_REPLACE) &&
++		!(n->nlmsg_flags & NLM_F_EXCL));
++}
++
+ /*
+  * Create/change qdisc.
+  */
+-
+ static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
+ 			   struct netlink_ext_ack *extack)
+ {
+@@ -1647,27 +1665,35 @@ replay:
+ 				 *
+ 				 *   We know, that some child q is already
+ 				 *   attached to this parent and have choice:
+-				 *   either to change it or to create/graft new one.
++				 *   1) change it or 2) create/graft new one.
++				 *   If the requested qdisc kind is different
++				 *   than the existing one, then we choose graft.
++				 *   If they are the same then this is "change"
++				 *   operation - just let it fallthrough..
+ 				 *
+ 				 *   1. We are allowed to create/graft only
+-				 *   if CREATE and REPLACE flags are set.
++				 *   if the request is explicitly stating
++				 *   "please create if it doesn't exist".
+ 				 *
+-				 *   2. If EXCL is set, requestor wanted to say,
+-				 *   that qdisc tcm_handle is not expected
++				 *   2. If the request is to exclusive create
++				 *   then the qdisc tcm_handle is not expected
+ 				 *   to exist, so that we choose create/graft too.
+ 				 *
+ 				 *   3. The last case is when no flags are set.
++				 *   This will happen when for example tc
++				 *   utility issues a "change" command.
+ 				 *   Alas, it is sort of hole in API, we
+ 				 *   cannot decide what to do unambiguously.
+-				 *   For now we select create/graft, if
+-				 *   user gave KIND, which does not match existing.
++				 *   For now we select create/graft.
+ 				 */
+-				if ((n->nlmsg_flags & NLM_F_CREATE) &&
+-				    (n->nlmsg_flags & NLM_F_REPLACE) &&
+-				    ((n->nlmsg_flags & NLM_F_EXCL) ||
+-				     (tca[TCA_KIND] &&
+-				      nla_strcmp(tca[TCA_KIND], q->ops->id))))
+-					goto create_n_graft;
++				if (tca[TCA_KIND] &&
++				    nla_strcmp(tca[TCA_KIND], q->ops->id)) {
++					if (req_create_or_replace(n) ||
++					    req_create_exclusive(n))
++						goto create_n_graft;
++					else if (req_change(n))
++						goto create_n_graft2;
++				}
+ 			}
+ 		}
+ 	} else {
+@@ -1701,6 +1727,7 @@ create_n_graft:
+ 		NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag");
+ 		return -ENOENT;
+ 	}
++create_n_graft2:
+ 	if (clid == TC_H_INGRESS) {
+ 		if (dev_ingress_queue(dev)) {
+ 			q = qdisc_create(dev, dev_ingress_queue(dev),
+diff --git a/net/sctp/socket.c b/net/sctp/socket.c
+index c806d272107ac..a11b0d903514c 100644
+--- a/net/sctp/socket.c
++++ b/net/sctp/socket.c
+@@ -98,7 +98,7 @@ struct percpu_counter sctp_sockets_allocated;
+ 
+ static void sctp_enter_memory_pressure(struct sock *sk)
+ {
+-	sctp_memory_pressure = 1;
++	WRITE_ONCE(sctp_memory_pressure, 1);
+ }
+ 
+ 
+@@ -9472,7 +9472,7 @@ void sctp_copy_sock(struct sock *newsk, struct sock *sk,
+ 	newinet->inet_rcv_saddr = inet->inet_rcv_saddr;
+ 	newinet->inet_dport = htons(asoc->peer.port);
+ 	newinet->pmtudisc = inet->pmtudisc;
+-	newinet->inet_id = get_random_u16();
++	atomic_set(&newinet->inet_id, get_random_u16());
+ 
+ 	newinet->uc_ttl = inet->uc_ttl;
+ 	newinet->mc_loop = 1;
+diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
+index b098fde373abf..28c0771c4e8c3 100644
+--- a/net/sunrpc/xprtrdma/verbs.c
++++ b/net/sunrpc/xprtrdma/verbs.c
+@@ -935,9 +935,6 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,
+ 	if (!rep->rr_rdmabuf)
+ 		goto out_free;
+ 
+-	if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))
+-		goto out_free_regbuf;
+-
+ 	rep->rr_cid.ci_completion_id =
+ 		atomic_inc_return(&r_xprt->rx_ep->re_completion_ids);
+ 
+@@ -956,8 +953,6 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,
+ 	spin_unlock(&buf->rb_lock);
+ 	return rep;
+ 
+-out_free_regbuf:
+-	rpcrdma_regbuf_free(rep->rr_rdmabuf);
+ out_free:
+ 	kfree(rep);
+ out:
+@@ -1363,6 +1358,10 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)
+ 			rep = rpcrdma_rep_create(r_xprt, temp);
+ 		if (!rep)
+ 			break;
++		if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) {
++			rpcrdma_rep_put(buf, rep);
++			break;
++		}
+ 
+ 		rep->rr_cid.ci_queue_id = ep->re_attr.recv_cq->res.id;
+ 		trace_xprtrdma_post_recv(rep);
+diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c
+index adcfb63b3550d..6f9ff4643dcbc 100644
+--- a/security/selinux/ss/policydb.c
++++ b/security/selinux/ss/policydb.c
+@@ -2005,6 +2005,7 @@ static int filename_trans_read_helper(struct policydb *p, void *fp)
+ 		if (!datum)
+ 			goto out;
+ 
++		datum->next = NULL;
+ 		*dst = datum;
+ 
+ 		/* ebitmap_read() will at least init the bitmap */
+@@ -2017,7 +2018,6 @@ static int filename_trans_read_helper(struct policydb *p, void *fp)
+ 			goto out;
+ 
+ 		datum->otype = le32_to_cpu(buf[0]);
+-		datum->next = NULL;
+ 
+ 		dst = &datum->next;
+ 	}
+diff --git a/sound/pci/ymfpci/ymfpci.c b/sound/pci/ymfpci/ymfpci.c
+index 82d4e0fda91be..d62a0e2ddf609 100644
+--- a/sound/pci/ymfpci/ymfpci.c
++++ b/sound/pci/ymfpci/ymfpci.c
+@@ -150,8 +150,8 @@ static inline int snd_ymfpci_create_gameport(struct snd_ymfpci *chip, int dev, i
+ void snd_ymfpci_free_gameport(struct snd_ymfpci *chip) { }
+ #endif /* SUPPORT_JOYSTICK */
+ 
+-static int snd_card_ymfpci_probe(struct pci_dev *pci,
+-				 const struct pci_device_id *pci_id)
++static int __snd_card_ymfpci_probe(struct pci_dev *pci,
++				   const struct pci_device_id *pci_id)
+ {
+ 	static int dev;
+ 	struct snd_card *card;
+@@ -333,6 +333,12 @@ static int snd_card_ymfpci_probe(struct pci_dev *pci,
+ 	return 0;
+ }
+ 
++static int snd_card_ymfpci_probe(struct pci_dev *pci,
++				 const struct pci_device_id *pci_id)
++{
++	return snd_card_free_on_error(&pci->dev, __snd_card_ymfpci_probe(pci, pci_id));
++}
++
+ static struct pci_driver ymfpci_driver = {
+ 	.name = KBUILD_MODNAME,
+ 	.id_table = snd_ymfpci_ids,
+diff --git a/sound/soc/amd/Kconfig b/sound/soc/amd/Kconfig
+index 3968c478c9381..44d4e6e51a358 100644
+--- a/sound/soc/amd/Kconfig
++++ b/sound/soc/amd/Kconfig
+@@ -71,6 +71,7 @@ config SND_SOC_AMD_RENOIR_MACH
+ config SND_SOC_AMD_ACP5x
+ 	tristate "AMD Audio Coprocessor-v5.x I2S support"
+ 	depends on X86 && PCI
++	select SND_AMD_ACP_CONFIG
+ 	help
+ 	 This option enables ACP v5.x support on AMD platform
+ 
+diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
+index c1ca3ceac5f2f..9a9571c3f08c0 100644
+--- a/sound/soc/amd/yc/acp6x-mach.c
++++ b/sound/soc/amd/yc/acp6x-mach.c
+@@ -217,7 +217,7 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
+ 		.driver_data = &acp6x_card,
+ 		.matches = {
+ 			DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"),
+-			DMI_MATCH(DMI_PRODUCT_NAME, "82"),
++			DMI_MATCH(DMI_PRODUCT_NAME, "82V2"),
+ 		}
+ 	},
+ 	{
+@@ -248,6 +248,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
+ 			DMI_MATCH(DMI_PRODUCT_NAME, "Redmi Book Pro 14 2022"),
+ 		}
+ 	},
++	{
++		.driver_data = &acp6x_card,
++		.matches = {
++			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
++			DMI_MATCH(DMI_PRODUCT_NAME, "M6500RC"),
++		}
++	},
+ 	{
+ 		.driver_data = &acp6x_card,
+ 		.matches = {
+diff --git a/sound/soc/codecs/cs35l41.c b/sound/soc/codecs/cs35l41.c
+index f2b5032daa6ae..2f4b0ee93aced 100644
+--- a/sound/soc/codecs/cs35l41.c
++++ b/sound/soc/codecs/cs35l41.c
+@@ -167,7 +167,7 @@ static int cs35l41_get_fs_mon_config_index(int freq)
+ static const DECLARE_TLV_DB_RANGE(dig_vol_tlv,
+ 		0, 0, TLV_DB_SCALE_ITEM(TLV_DB_GAIN_MUTE, 0, 1),
+ 		1, 913, TLV_DB_MINMAX_ITEM(-10200, 1200));
+-static DECLARE_TLV_DB_SCALE(amp_gain_tlv, 0, 1, 1);
++static DECLARE_TLV_DB_SCALE(amp_gain_tlv, 50, 100, 0);
+ 
+ static const struct snd_kcontrol_new dre_ctrl =
+ 	SOC_DAPM_SINGLE("Switch", CS35L41_PWR_CTRL3, 20, 1, 0);
+diff --git a/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh b/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh
+index 47ab90596acb2..6358df5752f90 100755
+--- a/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh
++++ b/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh
+@@ -57,8 +57,8 @@ ip link add name veth2-bond type veth peer name veth2-end
+ 
+ # add ports
+ ip link set fbond master fab-br0
+-ip link set veth1-bond down master fbond
+-ip link set veth2-bond down master fbond
++ip link set veth1-bond master fbond
++ip link set veth2-bond master fbond
+ 
+ # bring up
+ ip link set veth1-end up
+diff --git a/tools/testing/selftests/drivers/net/mlxsw/sharedbuffer.sh b/tools/testing/selftests/drivers/net/mlxsw/sharedbuffer.sh
+index 7d9e73a43a49b..0c47faff9274b 100755
+--- a/tools/testing/selftests/drivers/net/mlxsw/sharedbuffer.sh
++++ b/tools/testing/selftests/drivers/net/mlxsw/sharedbuffer.sh
+@@ -98,12 +98,12 @@ sb_occ_etc_check()
+ 
+ port_pool_test()
+ {
+-	local exp_max_occ=288
++	local exp_max_occ=$(devlink_cell_size_get)
+ 	local max_occ
+ 
+ 	devlink sb occupancy clearmax $DEVLINK_DEV
+ 
+-	$MZ $h1 -c 1 -p 160 -a $h1mac -b $h2mac -A 192.0.1.1 -B 192.0.1.2 \
++	$MZ $h1 -c 1 -p 10 -a $h1mac -b $h2mac -A 192.0.1.1 -B 192.0.1.2 \
+ 		-t ip -q
+ 
+ 	devlink sb occupancy snapshot $DEVLINK_DEV
+@@ -126,12 +126,12 @@ port_pool_test()
+ 
+ port_tc_ip_test()
+ {
+-	local exp_max_occ=288
++	local exp_max_occ=$(devlink_cell_size_get)
+ 	local max_occ
+ 
+ 	devlink sb occupancy clearmax $DEVLINK_DEV
+ 
+-	$MZ $h1 -c 1 -p 160 -a $h1mac -b $h2mac -A 192.0.1.1 -B 192.0.1.2 \
++	$MZ $h1 -c 1 -p 10 -a $h1mac -b $h2mac -A 192.0.1.1 -B 192.0.1.2 \
+ 		-t ip -q
+ 
+ 	devlink sb occupancy snapshot $DEVLINK_DEV
+@@ -154,16 +154,12 @@ port_tc_ip_test()
+ 
+ port_tc_arp_test()
+ {
+-	local exp_max_occ=96
++	local exp_max_occ=$(devlink_cell_size_get)
+ 	local max_occ
+ 
+-	if [[ $MLXSW_CHIP != "mlxsw_spectrum" ]]; then
+-		exp_max_occ=144
+-	fi
+-
+ 	devlink sb occupancy clearmax $DEVLINK_DEV
+ 
+-	$MZ $h1 -c 1 -p 160 -a $h1mac -A 192.0.1.1 -t arp -q
++	$MZ $h1 -c 1 -p 10 -a $h1mac -A 192.0.1.1 -t arp -q
+ 
+ 	devlink sb occupancy snapshot $DEVLINK_DEV
+ 
+diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
+index 69c58362c0edf..48d1a68be1d52 100644
+--- a/tools/testing/selftests/net/Makefile
++++ b/tools/testing/selftests/net/Makefile
+@@ -71,14 +71,60 @@ TEST_GEN_FILES += bind_bhash
+ TEST_GEN_PROGS += sk_bind_sendto_listen
+ TEST_GEN_PROGS += sk_connect_zero_addr
+ TEST_PROGS += test_ingress_egress_chaining.sh
++TEST_GEN_FILES += nat6to4.o
+ 
+ TEST_FILES := settings
+ 
+ include ../lib.mk
+ 
+-include bpf/Makefile
+-
+ $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
+ $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread
+ $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
+ $(OUTPUT)/bind_bhash: LDLIBS += -lpthread
++
++# Rules to generate bpf obj nat6to4.o
++CLANG ?= clang
++SCRATCH_DIR := $(OUTPUT)/tools
++BUILD_DIR := $(SCRATCH_DIR)/build
++BPFDIR := $(abspath ../../../lib/bpf)
++APIDIR := $(abspath ../../../include/uapi)
++
++CCINCLUDE += -I../bpf
++CCINCLUDE += -I../../../../usr/include/
++CCINCLUDE += -I$(SCRATCH_DIR)/include
++
++BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a
++
++MAKE_DIRS := $(BUILD_DIR)/libbpf
++$(MAKE_DIRS):
++	mkdir -p $@
++
++# Get Clang's default includes on this system, as opposed to those seen by
++# '-target bpf'. This fixes "missing" files on some architectures/distros,
++# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
++#
++# Use '-idirafter': Don't interfere with include mechanics except where the
++# build would have failed anyways.
++define get_sys_includes
++$(shell $(1) $(2) -v -E - </dev/null 2>&1 \
++	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
++$(shell $(1) $(2) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
++endef
++
++ifneq ($(CROSS_COMPILE),)
++CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
++endif
++
++CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
++
++$(OUTPUT)/nat6to4.o: nat6to4.c $(BPFOBJ) | $(MAKE_DIRS)
++	$(CLANG) -O2 -target bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@
++
++$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile)		       \
++	   $(APIDIR)/linux/bpf.h					       \
++	   | $(BUILD_DIR)/libbpf
++	$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/     \
++		    EXTRA_CFLAGS='-g -O0'				       \
++		    DESTDIR=$(SCRATCH_DIR) prefix= all install_headers
++
++EXTRA_CLEAN := $(SCRATCH_DIR)
+diff --git a/tools/testing/selftests/net/bpf/Makefile b/tools/testing/selftests/net/bpf/Makefile
+deleted file mode 100644
+index 8ccaf8732eb22..0000000000000
+--- a/tools/testing/selftests/net/bpf/Makefile
++++ /dev/null
+@@ -1,14 +0,0 @@
+-# SPDX-License-Identifier: GPL-2.0
+-
+-CLANG ?= clang
+-CCINCLUDE += -I../../bpf
+-CCINCLUDE += -I../../../../lib
+-CCINCLUDE += -I../../../../../usr/include/
+-
+-TEST_CUSTOM_PROGS = $(OUTPUT)/bpf/nat6to4.o
+-all: $(TEST_CUSTOM_PROGS)
+-
+-$(OUTPUT)/%.o: %.c
+-	$(CLANG) -O2 -target bpf -c $< $(CCINCLUDE) -o $@
+-
+-EXTRA_CLEAN := $(TEST_CUSTOM_PROGS)
+diff --git a/tools/testing/selftests/net/bpf/nat6to4.c b/tools/testing/selftests/net/bpf/nat6to4.c
+deleted file mode 100644
+index ac54c36b25fc8..0000000000000
+--- a/tools/testing/selftests/net/bpf/nat6to4.c
++++ /dev/null
+@@ -1,285 +0,0 @@
+-// SPDX-License-Identifier: GPL-2.0-only
+-/*
+- * This code is taken from the Android Open Source Project and the author
+- * (Maciej Żenczykowski) has gave permission to relicense it under the
+- * GPLv2. Therefore this program is free software;
+- * You can redistribute it and/or modify it under the terms of the GNU
+- * General Public License version 2 as published by the Free Software
+- * Foundation
+-
+- * The original headers, including the original license headers, are
+- * included below for completeness.
+- *
+- * Copyright (C) 2019 The Android Open Source Project
+- *
+- * Licensed under the Apache License, Version 2.0 (the "License");
+- * you may not use this file except in compliance with the License.
+- * You may obtain a copy of the License at
+- *
+- *      http://www.apache.org/licenses/LICENSE-2.0
+- *
+- * Unless required by applicable law or agreed to in writing, software
+- * distributed under the License is distributed on an "AS IS" BASIS,
+- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+- * See the License for the specific language governing permissions and
+- * limitations under the License.
+- */
+-#include <linux/bpf.h>
+-#include <linux/if.h>
+-#include <linux/if_ether.h>
+-#include <linux/if_packet.h>
+-#include <linux/in.h>
+-#include <linux/in6.h>
+-#include <linux/ip.h>
+-#include <linux/ipv6.h>
+-#include <linux/pkt_cls.h>
+-#include <linux/swab.h>
+-#include <stdbool.h>
+-#include <stdint.h>
+-
+-
+-#include <linux/udp.h>
+-
+-#include <bpf/bpf_helpers.h>
+-#include <bpf/bpf_endian.h>
+-
+-#define IP_DF 0x4000  // Flag: "Don't Fragment"
+-
+-SEC("schedcls/ingress6/nat_6")
+-int sched_cls_ingress6_nat_6_prog(struct __sk_buff *skb)
+-{
+-	const int l2_header_size =  sizeof(struct ethhdr);
+-	void *data = (void *)(long)skb->data;
+-	const void *data_end = (void *)(long)skb->data_end;
+-	const struct ethhdr * const eth = data;  // used iff is_ethernet
+-	const struct ipv6hdr * const ip6 =  (void *)(eth + 1);
+-
+-	// Require ethernet dst mac address to be our unicast address.
+-	if  (skb->pkt_type != PACKET_HOST)
+-		return TC_ACT_OK;
+-
+-	// Must be meta-ethernet IPv6 frame
+-	if (skb->protocol != bpf_htons(ETH_P_IPV6))
+-		return TC_ACT_OK;
+-
+-	// Must have (ethernet and) ipv6 header
+-	if (data + l2_header_size + sizeof(*ip6) > data_end)
+-		return TC_ACT_OK;
+-
+-	// Ethertype - if present - must be IPv6
+-	if (eth->h_proto != bpf_htons(ETH_P_IPV6))
+-		return TC_ACT_OK;
+-
+-	// IP version must be 6
+-	if (ip6->version != 6)
+-		return TC_ACT_OK;
+-	// Maximum IPv6 payload length that can be translated to IPv4
+-	if (bpf_ntohs(ip6->payload_len) > 0xFFFF - sizeof(struct iphdr))
+-		return TC_ACT_OK;
+-	switch (ip6->nexthdr) {
+-	case IPPROTO_TCP:  // For TCP & UDP the checksum neutrality of the chosen IPv6
+-	case IPPROTO_UDP:  // address means there is no need to update their checksums.
+-	case IPPROTO_GRE:  // We do not need to bother looking at GRE/ESP headers,
+-	case IPPROTO_ESP:  // since there is never a checksum to update.
+-		break;
+-	default:  // do not know how to handle anything else
+-		return TC_ACT_OK;
+-	}
+-
+-	struct ethhdr eth2;  // used iff is_ethernet
+-
+-	eth2 = *eth;                     // Copy over the ethernet header (src/dst mac)
+-	eth2.h_proto = bpf_htons(ETH_P_IP);  // But replace the ethertype
+-
+-	struct iphdr ip = {
+-		.version = 4,                                                      // u4
+-		.ihl = sizeof(struct iphdr) / sizeof(__u32),                       // u4
+-		.tos = (ip6->priority << 4) + (ip6->flow_lbl[0] >> 4),             // u8
+-		.tot_len = bpf_htons(bpf_ntohs(ip6->payload_len) + sizeof(struct iphdr)),  // u16
+-		.id = 0,                                                           // u16
+-		.frag_off = bpf_htons(IP_DF),                                          // u16
+-		.ttl = ip6->hop_limit,                                             // u8
+-		.protocol = ip6->nexthdr,                                          // u8
+-		.check = 0,                                                        // u16
+-		.saddr = 0x0201a8c0,                            // u32
+-		.daddr = 0x0101a8c0,                                         // u32
+-	};
+-
+-	// Calculate the IPv4 one's complement checksum of the IPv4 header.
+-	__wsum sum4 = 0;
+-
+-	for (int i = 0; i < sizeof(ip) / sizeof(__u16); ++i)
+-		sum4 += ((__u16 *)&ip)[i];
+-
+-	// Note that sum4 is guaranteed to be non-zero by virtue of ip.version == 4
+-	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse u32 into range 1 .. 0x1FFFE
+-	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse any potential carry into u16
+-	ip.check = (__u16)~sum4;                // sum4 cannot be zero, so this is never 0xFFFF
+-
+-	// Calculate the *negative* IPv6 16-bit one's complement checksum of the IPv6 header.
+-	__wsum sum6 = 0;
+-	// We'll end up with a non-zero sum due to ip6->version == 6 (which has '0' bits)
+-	for (int i = 0; i < sizeof(*ip6) / sizeof(__u16); ++i)
+-		sum6 += ~((__u16 *)ip6)[i];  // note the bitwise negation
+-
+-	// Note that there is no L4 checksum update: we are relying on the checksum neutrality
+-	// of the ipv6 address chosen by netd's ClatdController.
+-
+-	// Packet mutations begin - point of no return, but if this first modification fails
+-	// the packet is probably still pristine, so let clatd handle it.
+-	if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0))
+-		return TC_ACT_OK;
+-	bpf_csum_update(skb, sum6);
+-
+-	data = (void *)(long)skb->data;
+-	data_end = (void *)(long)skb->data_end;
+-	if (data + l2_header_size + sizeof(struct iphdr) > data_end)
+-		return TC_ACT_SHOT;
+-
+-	struct ethhdr *new_eth = data;
+-
+-	// Copy over the updated ethernet header
+-	*new_eth = eth2;
+-
+-	// Copy over the new ipv4 header.
+-	*(struct iphdr *)(new_eth + 1) = ip;
+-	return bpf_redirect(skb->ifindex, BPF_F_INGRESS);
+-}
+-
+-SEC("schedcls/egress4/snat4")
+-int sched_cls_egress4_snat4_prog(struct __sk_buff *skb)
+-{
+-	const int l2_header_size =  sizeof(struct ethhdr);
+-	void *data = (void *)(long)skb->data;
+-	const void *data_end = (void *)(long)skb->data_end;
+-	const struct ethhdr *const eth = data;  // used iff is_ethernet
+-	const struct iphdr *const ip4 = (void *)(eth + 1);
+-
+-	// Must be meta-ethernet IPv4 frame
+-	if (skb->protocol != bpf_htons(ETH_P_IP))
+-		return TC_ACT_OK;
+-
+-	// Must have ipv4 header
+-	if (data + l2_header_size + sizeof(struct ipv6hdr) > data_end)
+-		return TC_ACT_OK;
+-
+-	// Ethertype - if present - must be IPv4
+-	if (eth->h_proto != bpf_htons(ETH_P_IP))
+-		return TC_ACT_OK;
+-
+-	// IP version must be 4
+-	if (ip4->version != 4)
+-		return TC_ACT_OK;
+-
+-	// We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
+-	if (ip4->ihl != 5)
+-		return TC_ACT_OK;
+-
+-	// Maximum IPv6 payload length that can be translated to IPv4
+-	if (bpf_htons(ip4->tot_len) > 0xFFFF - sizeof(struct ipv6hdr))
+-		return TC_ACT_OK;
+-
+-	// Calculate the IPv4 one's complement checksum of the IPv4 header.
+-	__wsum sum4 = 0;
+-
+-	for (int i = 0; i < sizeof(*ip4) / sizeof(__u16); ++i)
+-		sum4 += ((__u16 *)ip4)[i];
+-
+-	// Note that sum4 is guaranteed to be non-zero by virtue of ip4->version == 4
+-	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse u32 into range 1 .. 0x1FFFE
+-	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse any potential carry into u16
+-	// for a correct checksum we should get *a* zero, but sum4 must be positive, ie 0xFFFF
+-	if (sum4 != 0xFFFF)
+-		return TC_ACT_OK;
+-
+-	// Minimum IPv4 total length is the size of the header
+-	if (bpf_ntohs(ip4->tot_len) < sizeof(*ip4))
+-		return TC_ACT_OK;
+-
+-	// We are incapable of dealing with IPv4 fragments
+-	if (ip4->frag_off & ~bpf_htons(IP_DF))
+-		return TC_ACT_OK;
+-
+-	switch (ip4->protocol) {
+-	case IPPROTO_TCP:  // For TCP & UDP the checksum neutrality of the chosen IPv6
+-	case IPPROTO_GRE:  // address means there is no need to update their checksums.
+-	case IPPROTO_ESP:  // We do not need to bother looking at GRE/ESP headers,
+-		break;         // since there is never a checksum to update.
+-
+-	case IPPROTO_UDP:  // See above comment, but must also have UDP header...
+-		if (data + sizeof(*ip4) + sizeof(struct udphdr) > data_end)
+-			return TC_ACT_OK;
+-		const struct udphdr *uh = (const struct udphdr *)(ip4 + 1);
+-		// If IPv4/UDP checksum is 0 then fallback to clatd so it can calculate the
+-		// checksum.  Otherwise the network or more likely the NAT64 gateway might
+-		// drop the packet because in most cases IPv6/UDP packets with a zero checksum
+-		// are invalid. See RFC 6935.  TODO: calculate checksum via bpf_csum_diff()
+-		if (!uh->check)
+-			return TC_ACT_OK;
+-		break;
+-
+-	default:  // do not know how to handle anything else
+-		return TC_ACT_OK;
+-	}
+-	struct ethhdr eth2;  // used iff is_ethernet
+-
+-	eth2 = *eth;                     // Copy over the ethernet header (src/dst mac)
+-	eth2.h_proto = bpf_htons(ETH_P_IPV6);  // But replace the ethertype
+-
+-	struct ipv6hdr ip6 = {
+-		.version = 6,                                    // __u8:4
+-		.priority = ip4->tos >> 4,                       // __u8:4
+-		.flow_lbl = {(ip4->tos & 0xF) << 4, 0, 0},       // __u8[3]
+-		.payload_len = bpf_htons(bpf_ntohs(ip4->tot_len) - 20),  // __be16
+-		.nexthdr = ip4->protocol,                        // __u8
+-		.hop_limit = ip4->ttl,                           // __u8
+-	};
+-	ip6.saddr.in6_u.u6_addr32[0] = bpf_htonl(0x20010db8);
+-	ip6.saddr.in6_u.u6_addr32[1] = 0;
+-	ip6.saddr.in6_u.u6_addr32[2] = 0;
+-	ip6.saddr.in6_u.u6_addr32[3] = bpf_htonl(1);
+-	ip6.daddr.in6_u.u6_addr32[0] = bpf_htonl(0x20010db8);
+-	ip6.daddr.in6_u.u6_addr32[1] = 0;
+-	ip6.daddr.in6_u.u6_addr32[2] = 0;
+-	ip6.daddr.in6_u.u6_addr32[3] = bpf_htonl(2);
+-
+-	// Calculate the IPv6 16-bit one's complement checksum of the IPv6 header.
+-	__wsum sum6 = 0;
+-	// We'll end up with a non-zero sum due to ip6.version == 6
+-	for (int i = 0; i < sizeof(ip6) / sizeof(__u16); ++i)
+-		sum6 += ((__u16 *)&ip6)[i];
+-
+-	// Packet mutations begin - point of no return, but if this first modification fails
+-	// the packet is probably still pristine, so let clatd handle it.
+-	if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0))
+-		return TC_ACT_OK;
+-
+-	// This takes care of updating the skb->csum field for a CHECKSUM_COMPLETE packet.
+-	// In such a case, skb->csum is a 16-bit one's complement sum of the entire payload,
+-	// thus we need to subtract out the ipv4 header's sum, and add in the ipv6 header's sum.
+-	// However, we've already verified the ipv4 checksum is correct and thus 0.
+-	// Thus we only need to add the ipv6 header's sum.
+-	//
+-	// bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
+-	// (-ENOTSUPP) if it isn't.  So we just ignore the return code (see above for more details).
+-	bpf_csum_update(skb, sum6);
+-
+-	// bpf_skb_change_proto() invalidates all pointers - reload them.
+-	data = (void *)(long)skb->data;
+-	data_end = (void *)(long)skb->data_end;
+-
+-	// I cannot think of any valid way for this error condition to trigger, however I do
+-	// believe the explicit check is required to keep the in kernel ebpf verifier happy.
+-	if (data + l2_header_size + sizeof(ip6) > data_end)
+-		return TC_ACT_SHOT;
+-
+-	struct ethhdr *new_eth = data;
+-
+-	// Copy over the updated ethernet header
+-	*new_eth = eth2;
+-	// Copy over the new ipv4 header.
+-	*(struct ipv6hdr *)(new_eth + 1) = ip6;
+-	return TC_ACT_OK;
+-}
+-
+-char _license[] SEC("license") = ("GPL");
+diff --git a/tools/testing/selftests/net/nat6to4.c b/tools/testing/selftests/net/nat6to4.c
+new file mode 100644
+index 0000000000000..ac54c36b25fc8
+--- /dev/null
++++ b/tools/testing/selftests/net/nat6to4.c
+@@ -0,0 +1,285 @@
++// SPDX-License-Identifier: GPL-2.0-only
++/*
++ * This code is taken from the Android Open Source Project and the author
++ * (Maciej Żenczykowski) has gave permission to relicense it under the
++ * GPLv2. Therefore this program is free software;
++ * You can redistribute it and/or modify it under the terms of the GNU
++ * General Public License version 2 as published by the Free Software
++ * Foundation
++
++ * The original headers, including the original license headers, are
++ * included below for completeness.
++ *
++ * Copyright (C) 2019 The Android Open Source Project
++ *
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++#include <linux/bpf.h>
++#include <linux/if.h>
++#include <linux/if_ether.h>
++#include <linux/if_packet.h>
++#include <linux/in.h>
++#include <linux/in6.h>
++#include <linux/ip.h>
++#include <linux/ipv6.h>
++#include <linux/pkt_cls.h>
++#include <linux/swab.h>
++#include <stdbool.h>
++#include <stdint.h>
++
++
++#include <linux/udp.h>
++
++#include <bpf/bpf_helpers.h>
++#include <bpf/bpf_endian.h>
++
++#define IP_DF 0x4000  // Flag: "Don't Fragment"
++
++SEC("schedcls/ingress6/nat_6")
++int sched_cls_ingress6_nat_6_prog(struct __sk_buff *skb)
++{
++	const int l2_header_size =  sizeof(struct ethhdr);
++	void *data = (void *)(long)skb->data;
++	const void *data_end = (void *)(long)skb->data_end;
++	const struct ethhdr * const eth = data;  // used iff is_ethernet
++	const struct ipv6hdr * const ip6 =  (void *)(eth + 1);
++
++	// Require ethernet dst mac address to be our unicast address.
++	if  (skb->pkt_type != PACKET_HOST)
++		return TC_ACT_OK;
++
++	// Must be meta-ethernet IPv6 frame
++	if (skb->protocol != bpf_htons(ETH_P_IPV6))
++		return TC_ACT_OK;
++
++	// Must have (ethernet and) ipv6 header
++	if (data + l2_header_size + sizeof(*ip6) > data_end)
++		return TC_ACT_OK;
++
++	// Ethertype - if present - must be IPv6
++	if (eth->h_proto != bpf_htons(ETH_P_IPV6))
++		return TC_ACT_OK;
++
++	// IP version must be 6
++	if (ip6->version != 6)
++		return TC_ACT_OK;
++	// Maximum IPv6 payload length that can be translated to IPv4
++	if (bpf_ntohs(ip6->payload_len) > 0xFFFF - sizeof(struct iphdr))
++		return TC_ACT_OK;
++	switch (ip6->nexthdr) {
++	case IPPROTO_TCP:  // For TCP & UDP the checksum neutrality of the chosen IPv6
++	case IPPROTO_UDP:  // address means there is no need to update their checksums.
++	case IPPROTO_GRE:  // We do not need to bother looking at GRE/ESP headers,
++	case IPPROTO_ESP:  // since there is never a checksum to update.
++		break;
++	default:  // do not know how to handle anything else
++		return TC_ACT_OK;
++	}
++
++	struct ethhdr eth2;  // used iff is_ethernet
++
++	eth2 = *eth;                     // Copy over the ethernet header (src/dst mac)
++	eth2.h_proto = bpf_htons(ETH_P_IP);  // But replace the ethertype
++
++	struct iphdr ip = {
++		.version = 4,                                                      // u4
++		.ihl = sizeof(struct iphdr) / sizeof(__u32),                       // u4
++		.tos = (ip6->priority << 4) + (ip6->flow_lbl[0] >> 4),             // u8
++		.tot_len = bpf_htons(bpf_ntohs(ip6->payload_len) + sizeof(struct iphdr)),  // u16
++		.id = 0,                                                           // u16
++		.frag_off = bpf_htons(IP_DF),                                          // u16
++		.ttl = ip6->hop_limit,                                             // u8
++		.protocol = ip6->nexthdr,                                          // u8
++		.check = 0,                                                        // u16
++		.saddr = 0x0201a8c0,                            // u32
++		.daddr = 0x0101a8c0,                                         // u32
++	};
++
++	// Calculate the IPv4 one's complement checksum of the IPv4 header.
++	__wsum sum4 = 0;
++
++	for (int i = 0; i < sizeof(ip) / sizeof(__u16); ++i)
++		sum4 += ((__u16 *)&ip)[i];
++
++	// Note that sum4 is guaranteed to be non-zero by virtue of ip.version == 4
++	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse u32 into range 1 .. 0x1FFFE
++	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse any potential carry into u16
++	ip.check = (__u16)~sum4;                // sum4 cannot be zero, so this is never 0xFFFF
++
++	// Calculate the *negative* IPv6 16-bit one's complement checksum of the IPv6 header.
++	__wsum sum6 = 0;
++	// We'll end up with a non-zero sum due to ip6->version == 6 (which has '0' bits)
++	for (int i = 0; i < sizeof(*ip6) / sizeof(__u16); ++i)
++		sum6 += ~((__u16 *)ip6)[i];  // note the bitwise negation
++
++	// Note that there is no L4 checksum update: we are relying on the checksum neutrality
++	// of the ipv6 address chosen by netd's ClatdController.
++
++	// Packet mutations begin - point of no return, but if this first modification fails
++	// the packet is probably still pristine, so let clatd handle it.
++	if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0))
++		return TC_ACT_OK;
++	bpf_csum_update(skb, sum6);
++
++	data = (void *)(long)skb->data;
++	data_end = (void *)(long)skb->data_end;
++	if (data + l2_header_size + sizeof(struct iphdr) > data_end)
++		return TC_ACT_SHOT;
++
++	struct ethhdr *new_eth = data;
++
++	// Copy over the updated ethernet header
++	*new_eth = eth2;
++
++	// Copy over the new ipv4 header.
++	*(struct iphdr *)(new_eth + 1) = ip;
++	return bpf_redirect(skb->ifindex, BPF_F_INGRESS);
++}
++
++SEC("schedcls/egress4/snat4")
++int sched_cls_egress4_snat4_prog(struct __sk_buff *skb)
++{
++	const int l2_header_size =  sizeof(struct ethhdr);
++	void *data = (void *)(long)skb->data;
++	const void *data_end = (void *)(long)skb->data_end;
++	const struct ethhdr *const eth = data;  // used iff is_ethernet
++	const struct iphdr *const ip4 = (void *)(eth + 1);
++
++	// Must be meta-ethernet IPv4 frame
++	if (skb->protocol != bpf_htons(ETH_P_IP))
++		return TC_ACT_OK;
++
++	// Must have ipv4 header
++	if (data + l2_header_size + sizeof(struct ipv6hdr) > data_end)
++		return TC_ACT_OK;
++
++	// Ethertype - if present - must be IPv4
++	if (eth->h_proto != bpf_htons(ETH_P_IP))
++		return TC_ACT_OK;
++
++	// IP version must be 4
++	if (ip4->version != 4)
++		return TC_ACT_OK;
++
++	// We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
++	if (ip4->ihl != 5)
++		return TC_ACT_OK;
++
++	// Maximum IPv6 payload length that can be translated to IPv4
++	if (bpf_htons(ip4->tot_len) > 0xFFFF - sizeof(struct ipv6hdr))
++		return TC_ACT_OK;
++
++	// Calculate the IPv4 one's complement checksum of the IPv4 header.
++	__wsum sum4 = 0;
++
++	for (int i = 0; i < sizeof(*ip4) / sizeof(__u16); ++i)
++		sum4 += ((__u16 *)ip4)[i];
++
++	// Note that sum4 is guaranteed to be non-zero by virtue of ip4->version == 4
++	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse u32 into range 1 .. 0x1FFFE
++	sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse any potential carry into u16
++	// for a correct checksum we should get *a* zero, but sum4 must be positive, ie 0xFFFF
++	if (sum4 != 0xFFFF)
++		return TC_ACT_OK;
++
++	// Minimum IPv4 total length is the size of the header
++	if (bpf_ntohs(ip4->tot_len) < sizeof(*ip4))
++		return TC_ACT_OK;
++
++	// We are incapable of dealing with IPv4 fragments
++	if (ip4->frag_off & ~bpf_htons(IP_DF))
++		return TC_ACT_OK;
++
++	switch (ip4->protocol) {
++	case IPPROTO_TCP:  // For TCP & UDP the checksum neutrality of the chosen IPv6
++	case IPPROTO_GRE:  // address means there is no need to update their checksums.
++	case IPPROTO_ESP:  // We do not need to bother looking at GRE/ESP headers,
++		break;         // since there is never a checksum to update.
++
++	case IPPROTO_UDP:  // See above comment, but must also have UDP header...
++		if (data + sizeof(*ip4) + sizeof(struct udphdr) > data_end)
++			return TC_ACT_OK;
++		const struct udphdr *uh = (const struct udphdr *)(ip4 + 1);
++		// If IPv4/UDP checksum is 0 then fallback to clatd so it can calculate the
++		// checksum.  Otherwise the network or more likely the NAT64 gateway might
++		// drop the packet because in most cases IPv6/UDP packets with a zero checksum
++		// are invalid. See RFC 6935.  TODO: calculate checksum via bpf_csum_diff()
++		if (!uh->check)
++			return TC_ACT_OK;
++		break;
++
++	default:  // do not know how to handle anything else
++		return TC_ACT_OK;
++	}
++	struct ethhdr eth2;  // used iff is_ethernet
++
++	eth2 = *eth;                     // Copy over the ethernet header (src/dst mac)
++	eth2.h_proto = bpf_htons(ETH_P_IPV6);  // But replace the ethertype
++
++	struct ipv6hdr ip6 = {
++		.version = 6,                                    // __u8:4
++		.priority = ip4->tos >> 4,                       // __u8:4
++		.flow_lbl = {(ip4->tos & 0xF) << 4, 0, 0},       // __u8[3]
++		.payload_len = bpf_htons(bpf_ntohs(ip4->tot_len) - 20),  // __be16
++		.nexthdr = ip4->protocol,                        // __u8
++		.hop_limit = ip4->ttl,                           // __u8
++	};
++	ip6.saddr.in6_u.u6_addr32[0] = bpf_htonl(0x20010db8);
++	ip6.saddr.in6_u.u6_addr32[1] = 0;
++	ip6.saddr.in6_u.u6_addr32[2] = 0;
++	ip6.saddr.in6_u.u6_addr32[3] = bpf_htonl(1);
++	ip6.daddr.in6_u.u6_addr32[0] = bpf_htonl(0x20010db8);
++	ip6.daddr.in6_u.u6_addr32[1] = 0;
++	ip6.daddr.in6_u.u6_addr32[2] = 0;
++	ip6.daddr.in6_u.u6_addr32[3] = bpf_htonl(2);
++
++	// Calculate the IPv6 16-bit one's complement checksum of the IPv6 header.
++	__wsum sum6 = 0;
++	// We'll end up with a non-zero sum due to ip6.version == 6
++	for (int i = 0; i < sizeof(ip6) / sizeof(__u16); ++i)
++		sum6 += ((__u16 *)&ip6)[i];
++
++	// Packet mutations begin - point of no return, but if this first modification fails
++	// the packet is probably still pristine, so let clatd handle it.
++	if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0))
++		return TC_ACT_OK;
++
++	// This takes care of updating the skb->csum field for a CHECKSUM_COMPLETE packet.
++	// In such a case, skb->csum is a 16-bit one's complement sum of the entire payload,
++	// thus we need to subtract out the ipv4 header's sum, and add in the ipv6 header's sum.
++	// However, we've already verified the ipv4 checksum is correct and thus 0.
++	// Thus we only need to add the ipv6 header's sum.
++	//
++	// bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
++	// (-ENOTSUPP) if it isn't.  So we just ignore the return code (see above for more details).
++	bpf_csum_update(skb, sum6);
++
++	// bpf_skb_change_proto() invalidates all pointers - reload them.
++	data = (void *)(long)skb->data;
++	data_end = (void *)(long)skb->data_end;
++
++	// I cannot think of any valid way for this error condition to trigger, however I do
++	// believe the explicit check is required to keep the in kernel ebpf verifier happy.
++	if (data + l2_header_size + sizeof(ip6) > data_end)
++		return TC_ACT_SHOT;
++
++	struct ethhdr *new_eth = data;
++
++	// Copy over the updated ethernet header
++	*new_eth = eth2;
++	// Copy over the new ipv4 header.
++	*(struct ipv6hdr *)(new_eth + 1) = ip6;
++	return TC_ACT_OK;
++}
++
++char _license[] SEC("license") = ("GPL");
+diff --git a/tools/testing/selftests/net/udpgro_frglist.sh b/tools/testing/selftests/net/udpgro_frglist.sh
+index c9c4b9d658390..0a6359bed0b92 100755
+--- a/tools/testing/selftests/net/udpgro_frglist.sh
++++ b/tools/testing/selftests/net/udpgro_frglist.sh
+@@ -40,8 +40,8 @@ run_one() {
+ 
+ 	ip -n "${PEER_NS}" link set veth1 xdp object ${BPF_FILE} section xdp
+ 	tc -n "${PEER_NS}" qdisc add dev veth1 clsact
+-	tc -n "${PEER_NS}" filter add dev veth1 ingress prio 4 protocol ipv6 bpf object-file ../bpf/nat6to4.o section schedcls/ingress6/nat_6  direct-action
+-	tc -n "${PEER_NS}" filter add dev veth1 egress prio 4 protocol ip bpf object-file ../bpf/nat6to4.o section schedcls/egress4/snat4 direct-action
++	tc -n "${PEER_NS}" filter add dev veth1 ingress prio 4 protocol ipv6 bpf object-file nat6to4.o section schedcls/ingress6/nat_6  direct-action
++	tc -n "${PEER_NS}" filter add dev veth1 egress prio 4 protocol ip bpf object-file nat6to4.o section schedcls/egress4/snat4 direct-action
+         echo ${rx_args}
+ 	ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} -r &
+ 
+@@ -88,8 +88,8 @@ if [ ! -f ${BPF_FILE} ]; then
+ 	exit -1
+ fi
+ 
+-if [ ! -f bpf/nat6to4.o ]; then
+-	echo "Missing nat6to4 helper. Build bpfnat6to4.o selftest first"
++if [ ! -f nat6to4.o ]; then
++	echo "Missing nat6to4 helper. Build bpf nat6to4.o selftest first"
+ 	exit -1
+ fi
+