diff --git a/data/meson.build b/data/meson.build
index 88e5879..8a49939 100644
--- a/data/meson.build
+++ b/data/meson.build
@@ -54,13 +54,21 @@ configure_file(
 )
 
 
-# Install the Polkit action file in all cases
-configure_file(
-    input: 'polkit/actions/com.feralinteractive.GameMode.policy.in',
-    output: 'com.feralinteractive.GameMode.policy',
-    configuration: data_conf,
-    install_dir: path_polkit_action_dir,
-)
+# Install the Polkit action & rule files for the privileged gamemode group
+if with_privileged_group != ''
+    configure_file(
+        input: 'polkit/actions/com.feralinteractive.GameMode.policy.in',
+        output: 'com.feralinteractive.GameMode.policy',
+        configuration: data_conf,
+        install_dir: path_polkit_action_dir,
+    )
+    configure_file(
+        input: 'polkit/rules.d/gamemode.rules.in',
+        output: 'gamemode.rules',
+        configuration: data_conf,
+        install_dir: path_polkit_rule_dir,
+    )
+endif
 
 # Install the helper run script
 install_data(
diff --git a/data/polkit/actions/com.feralinteractive.GameMode.policy.in b/data/polkit/actions/com.feralinteractive.GameMode.policy.in
index 9933e0b..c3e3a20 100644
--- a/data/polkit/actions/com.feralinteractive.GameMode.policy.in
+++ b/data/polkit/actions/com.feralinteractive.GameMode.policy.in
@@ -18,7 +18,7 @@
     <defaults>
       <allow_any>no</allow_any>
       <allow_inactive>no</allow_inactive>
-      <allow_active>yes</allow_active>
+      <allow_active>no</allow_active>
     </defaults>
     <annotate key="org.freedesktop.policykit.exec.path">@LIBEXECDIR@/cpugovctl</annotate>
   </action>
@@ -29,7 +29,7 @@
     <defaults>
       <allow_any>no</allow_any>
       <allow_inactive>no</allow_inactive>
-      <allow_active>yes</allow_active>
+      <allow_active>no</allow_active>
     </defaults>
     <annotate key="org.freedesktop.policykit.exec.path">@LIBEXECDIR@/gpuclockctl</annotate>
     <annotate key="org.freedesktop.policykit.exec.allow_gui">true</annotate>
diff --git a/data/polkit/rules.d/gamemode.rules.in b/data/polkit/rules.d/gamemode.rules.in
new file mode 100644
index 0000000..504e48c
--- /dev/null
+++ b/data/polkit/rules.d/gamemode.rules.in
@@ -0,0 +1,12 @@
+/*
+ * Allow users in privileged gamemode group to run cpugovctl &
+ * gpuclockctl without authentication
+ */
+polkit.addRule(function (action, subject) {
+    if ((action.id == "com.feralinteractive.GameMode.governor-helper" ||
+         action.id == "com.feralinteractive.GameMode.gpu-helper") &&
+        subject.isInGroup("@GAMEMODE_PRIVILEGED_GROUP@"))
+    {
+        return polkit.Result.YES;
+    }
+});
diff --git a/meson.build b/meson.build
index 77f8359..8672e82 100644
--- a/meson.build
+++ b/meson.build
@@ -142,7 +142,9 @@ if path_dbus_service_dir == ''
   path_dbus_service_dir = join_paths(path_datadir, 'dbus-1', 'services')
 endif
 
-path_polkit_action_dir = join_paths(path_datadir, 'polkit-1', 'actions')
+path_polkit_dir = join_paths(path_datadir, 'polkit-1')
+path_polkit_action_dir = join_paths(path_polkit_dir, 'actions')
+path_polkit_rule_dir = join_paths(path_polkit_dir, 'rules.d')
 
 with_examples = get_option('with-examples')
 with_util = get_option('with-util')