Skip to content

Commit 05bf58c

Browse files
torvaldstorvalds
committed
Merge branch 'sched-idle-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull sched/idle changes from Ingo Molnar: "More idle code reorganization, to prepare for more integration. (Sent separately because it depended on pending timer work, which is now upstream)" * 'sched-idle-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/idle: Add more comments to the code sched/idle: Move idle conditions in cpuidle_idle main function sched/idle: Reorganize the idle loop cpuidle/idle: Move the cpuidle_idle_call function to idle.c idle/cpuidle: Split cpuidle_idle_call main function into smaller functions
2 parents d230822 + a1d028b commit 05bf58c

File tree

3 files changed

+211
-69
lines changed

3 files changed

+211
-69
lines changed

‎drivers/cpuidle/cpuidle.c

Lines changed: 56 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,26 @@ int cpuidle_play_dead(void)
6464
return -ENODEV;
6565
}
6666

67+
/**
68+
* cpuidle_enabled - check if the cpuidle framework is ready
69+
* @dev: cpuidle device for this cpu
70+
* @drv: cpuidle driver for this cpu
71+
*
72+
* Return 0 on success, otherwise:
73+
* -NODEV : the cpuidle framework is not available
74+
* -EBUSY : the cpuidle framework is not initialized
75+
*/
76+
int cpuidle_enabled(struct cpuidle_driver *drv, struct cpuidle_device *dev)
77+
{
78+
if (off || !initialized)
79+
return -ENODEV;
80+
81+
if (!drv || !dev || !dev->enabled)
82+
return -EBUSY;
83+
84+
return 0;
85+
}
86+
6787
/**
6888
* cpuidle_enter_state - enter the state and update stats
6989
* @dev: cpuidle device for this cpu
@@ -109,63 +129,48 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
109129
}
110130

111131
/**
112-
* cpuidle_idle_call - the main idle loop
132+
* cpuidle_select - ask the cpuidle framework to choose an idle state
133+
*
134+
* @drv: the cpuidle driver
135+
* @dev: the cpuidle device
113136
*
114-
* NOTE: no locks or semaphores should be used here
115-
* return non-zero on failure
137+
* Returns the index of the idle state.
116138
*/
117-
int cpuidle_idle_call(void)
139+
int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
118140
{
119-
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
120-
struct cpuidle_driver *drv;
121-
int next_state, entered_state;
122-
bool broadcast;
123-
124-
if (off || !initialized)
125-
return -ENODEV;
126-
127-
/* check if the device is ready */
128-
if (!dev || !dev->enabled)
129-
return -EBUSY;
130-
131-
drv = cpuidle_get_cpu_driver(dev);
132-
133-
/* ask the governor for the next state */
134-
next_state = cpuidle_curr_governor->select(drv, dev);
135-
if (need_resched()) {
136-
dev->last_residency = 0;
137-
/* give the governor an opportunity to reflect on the outcome */
138-
if (cpuidle_curr_governor->reflect)
139-
cpuidle_curr_governor->reflect(dev, next_state);
140-
local_irq_enable();
141-
return 0;
142-
}
143-
144-
broadcast = !!(drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP);
145-
146-
if (broadcast &&
147-
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu))
148-
return -EBUSY;
149-
150-
151-
trace_cpu_idle_rcuidle(next_state, dev->cpu);
152-
153-
if (cpuidle_state_is_coupled(dev, drv, next_state))
154-
entered_state = cpuidle_enter_state_coupled(dev, drv,
155-
next_state);
156-
else
157-
entered_state = cpuidle_enter_state(dev, drv, next_state);
158-
159-
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
141+
return cpuidle_curr_governor->select(drv, dev);
142+
}
160143

161-
if (broadcast)
162-
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
144+
/**
145+
* cpuidle_enter - enter into the specified idle state
146+
*
147+
* @drv: the cpuidle driver tied with the cpu
148+
* @dev: the cpuidle device
149+
* @index: the index in the idle state table
150+
*
151+
* Returns the index in the idle state, < 0 in case of error.
152+
* The error code depends on the backend driver
153+
*/
154+
int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
155+
int index)
156+
{
157+
if (cpuidle_state_is_coupled(dev, drv, index))
158+
return cpuidle_enter_state_coupled(dev, drv, index);
159+
return cpuidle_enter_state(dev, drv, index);
160+
}
163161

164-
/* give the governor an opportunity to reflect on the outcome */
162+
/**
163+
* cpuidle_reflect - tell the underlying governor what was the state
164+
* we were in
165+
*
166+
* @dev : the cpuidle device
167+
* @index: the index in the idle state table
168+
*
169+
*/
170+
void cpuidle_reflect(struct cpuidle_device *dev, int index)
171+
{
165172
if (cpuidle_curr_governor->reflect)
166-
cpuidle_curr_governor->reflect(dev, entered_state);
167-
168-
return 0;
173+
cpuidle_curr_governor->reflect(dev, index);
169174
}
170175

171176
/**

‎include/linux/cpuidle.h

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,15 @@ struct cpuidle_driver {
119119

120120
#ifdef CONFIG_CPU_IDLE
121121
extern void disable_cpuidle(void);
122-
extern int cpuidle_idle_call(void);
122+
123+
extern int cpuidle_enabled(struct cpuidle_driver *drv,
124+
struct cpuidle_device *dev);
125+
extern int cpuidle_select(struct cpuidle_driver *drv,
126+
struct cpuidle_device *dev);
127+
extern int cpuidle_enter(struct cpuidle_driver *drv,
128+
struct cpuidle_device *dev, int index);
129+
extern void cpuidle_reflect(struct cpuidle_device *dev, int index);
130+
123131
extern int cpuidle_register_driver(struct cpuidle_driver *drv);
124132
extern struct cpuidle_driver *cpuidle_get_driver(void);
125133
extern struct cpuidle_driver *cpuidle_driver_ref(void);
@@ -141,7 +149,16 @@ extern int cpuidle_play_dead(void);
141149
extern struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev);
142150
#else
143151
static inline void disable_cpuidle(void) { }
144-
static inline int cpuidle_idle_call(void) { return -ENODEV; }
152+
static inline int cpuidle_enabled(struct cpuidle_driver *drv,
153+
struct cpuidle_device *dev)
154+
{return -ENODEV; }
155+
static inline int cpuidle_select(struct cpuidle_driver *drv,
156+
struct cpuidle_device *dev)
157+
{return -ENODEV; }
158+
static inline int cpuidle_enter(struct cpuidle_driver *drv,
159+
struct cpuidle_device *dev, int index)
160+
{return -ENODEV; }
161+
static inline void cpuidle_reflect(struct cpuidle_device *dev, int index) { }
145162
static inline int cpuidle_register_driver(struct cpuidle_driver *drv)
146163
{return -ENODEV; }
147164
static inline struct cpuidle_driver *cpuidle_get_driver(void) {return NULL; }
@@ -163,6 +180,8 @@ static inline int cpuidle_enable_device(struct cpuidle_device *dev)
163180
{return -ENODEV; }
164181
static inline void cpuidle_disable_device(struct cpuidle_device *dev) { }
165182
static inline int cpuidle_play_dead(void) {return -ENODEV; }
183+
static inline struct cpuidle_driver *cpuidle_get_cpu_driver(
184+
struct cpuidle_device *dev) {return NULL; }
166185
#endif
167186

168187
#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED

‎kernel/sched/idle.c

Lines changed: 134 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,136 @@ void __weak arch_cpu_idle(void)
6363
local_irq_enable();
6464
}
6565

66+
/**
67+
* cpuidle_idle_call - the main idle function
68+
*
69+
* NOTE: no locks or semaphores should be used here
70+
* return non-zero on failure
71+
*/
72+
static int cpuidle_idle_call(void)
73+
{
74+
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
75+
struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
76+
int next_state, entered_state, ret;
77+
bool broadcast;
78+
79+
/*
80+
* Check if the idle task must be rescheduled. If it is the
81+
* case, exit the function after re-enabling the local irq and
82+
* set again the polling flag
83+
*/
84+
if (current_clr_polling_and_test()) {
85+
local_irq_enable();
86+
__current_set_polling();
87+
return 0;
88+
}
89+
90+
/*
91+
* During the idle period, stop measuring the disabled irqs
92+
* critical sections latencies
93+
*/
94+
stop_critical_timings();
95+
96+
/*
97+
* Tell the RCU framework we are entering an idle section,
98+
* so no more rcu read side critical sections and one more
99+
* step to the grace period
100+
*/
101+
rcu_idle_enter();
102+
103+
/*
104+
* Check if the cpuidle framework is ready, otherwise fallback
105+
* to the default arch specific idle method
106+
*/
107+
ret = cpuidle_enabled(drv, dev);
108+
109+
if (!ret) {
110+
/*
111+
* Ask the governor to choose an idle state it thinks
112+
* it is convenient to go to. There is *always* a
113+
* convenient idle state
114+
*/
115+
next_state = cpuidle_select(drv, dev);
116+
117+
/*
118+
* The idle task must be scheduled, it is pointless to
119+
* go to idle, just update no idle residency and get
120+
* out of this function
121+
*/
122+
if (current_clr_polling_and_test()) {
123+
dev->last_residency = 0;
124+
entered_state = next_state;
125+
local_irq_enable();
126+
} else {
127+
broadcast = !!(drv->states[next_state].flags &
128+
CPUIDLE_FLAG_TIMER_STOP);
129+
130+
if (broadcast)
131+
/*
132+
* Tell the time framework to switch
133+
* to a broadcast timer because our
134+
* local timer will be shutdown. If a
135+
* local timer is used from another
136+
* cpu as a broadcast timer, this call
137+
* may fail if it is not available
138+
*/
139+
ret = clockevents_notify(
140+
CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
141+
&dev->cpu);
142+
143+
if (!ret) {
144+
trace_cpu_idle_rcuidle(next_state, dev->cpu);
145+
146+
/*
147+
* Enter the idle state previously
148+
* returned by the governor
149+
* decision. This function will block
150+
* until an interrupt occurs and will
151+
* take care of re-enabling the local
152+
* interrupts
153+
*/
154+
entered_state = cpuidle_enter(drv, dev,
155+
next_state);
156+
157+
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT,
158+
dev->cpu);
159+
160+
if (broadcast)
161+
clockevents_notify(
162+
CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
163+
&dev->cpu);
164+
165+
/*
166+
* Give the governor an opportunity to reflect on the
167+
* outcome
168+
*/
169+
cpuidle_reflect(dev, entered_state);
170+
}
171+
}
172+
}
173+
174+
/*
175+
* We can't use the cpuidle framework, let's use the default
176+
* idle routine
177+
*/
178+
if (ret)
179+
arch_cpu_idle();
180+
181+
__current_set_polling();
182+
183+
/*
184+
* It is up to the idle functions to enable back the local
185+
* interrupt
186+
*/
187+
if (WARN_ON_ONCE(irqs_disabled()))
188+
local_irq_enable();
189+
190+
rcu_idle_exit();
191+
start_critical_timings();
192+
193+
return 0;
194+
}
195+
66196
/*
67197
* Generic idle loop implementation
68198
*/
@@ -90,23 +220,11 @@ static void cpu_idle_loop(void)
90220
* know that the IPI is going to arrive right
91221
* away
92222
*/
93-
if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
223+
if (cpu_idle_force_poll || tick_check_broadcast_expired())
94224
cpu_idle_poll();
95-
} else {
96-
if (!current_clr_polling_and_test()) {
97-
stop_critical_timings();
98-
rcu_idle_enter();
99-
if (cpuidle_idle_call())
100-
arch_cpu_idle();
101-
if (WARN_ON_ONCE(irqs_disabled()))
102-
local_irq_enable();
103-
rcu_idle_exit();
104-
start_critical_timings();
105-
} else {
106-
local_irq_enable();
107-
}
108-
__current_set_polling();
109-
}
225+
else
226+
cpuidle_idle_call();
227+
110228
arch_cpu_idle_exit();
111229
}
112230

0 commit comments

Comments
 (0)