一、概述
系统进入suspended或进程被加入到cgroup冻结或解冻分组，用户进程和部分内核线程被冻结后，会剥夺执行cpu资源，解冻或唤醒后恢复正常。
二、进程冻结与解冻原理
2.1 进程冻结
用户进程和内核线程冻结的基本流程：
内核态线程可直接唤醒执行冻结操作，用户进程则需要在其返回到用户态时才能执行冻结操作。这是因为若其在内核态执行时被冻结，若其正好持有一些锁，则可能会导致死锁。为此，内核通过了一个比较巧妙的方式实现了冻结流程。它首先为该进程设置了一个信号pending标志TIF_SIGPENDING，但并不向该进程发送实际的信号，然后通过ipi唤醒该进程执行。由于ipi会进行进程内核的中断处理流程，当其处理完成后，会调用ret_to_user函数返回用户态，而该函数会调用信号处理函数检查是否有pending的中断需要处理，由于先前已经设置了信号的pending标志，因此会执行信号处理流程。此时，会发现进程冻结相关的全局变量已设置，故进程将执行冻结流程。
2.1.1 冻结用户进程
int freeze_processes(void)
{int error;error = __usermodehelper_disable(UMH_FREEZING);if (error)return error;/* Make sure this task doesn't get frozen */current->flags |= PF_SUSPEND_TASK;if (!pm_freezing)atomic_inc(&system_freezing_cnt);pm_wakeup_clear(0);pr_info("%s:Freezing user space processes ... ", STR_KERNEL_LOG_ENTER);pm_freezing = true;// true表示用户进程error = try_to_freeze_tasks(true);if (!error) {__usermodehelper_set_disable_depth(UMH_DISABLED);pr_cont("done.");}pr_cont("\n");BUG_ON(in_atomic());/** Now that the whole userspace is frozen we need to disable* the OOM killer to disallow any further interference with* killable tasks. There is no guarantee oom victims will* ever reach a point they go away we have to wait with a timeout.*/if (!error && !oom_killer_disable(msecs_to_jiffies(freeze_timeout_msecs)))error = -EBUSY;if (error)thaw_processes();return error;
}static int try_to_freeze_tasks(bool user_only) {if (!user_only)freeze_workqueues_begin();while (true) {todo = 0;read_lock(&tasklist_lock);for_each_process_thread(g, p) {if (p == current || !freeze_task(p))continue;......}......
}bool freeze_task(struct task_struct *p)
{unsigned long flags;/** This check can race with freezer_do_not_count, but worst case that* will result in an extra wakeup being sent to the task. It does not* race with freezer_count(), the barriers in freezer_count() and* freezer_should_skip() ensure that either freezer_count() sees* freezing == true in try_to_freeze() and freezes, or* freezer_should_skip() sees !PF_FREEZE_SKIP and freezes the task* normally.*/// 若进程设置了PF_FREEZER_SKIP，则不能冻结if (freezer_should_skip(p))return false;spin_lock_irqsave(&freezer_lock, flags);if (!freezing(p) || frozen(p)) {spin_unlock_irqrestore(&freezer_lock, flags);return false;}// 如果是用户进程，需要先发送伪信号，当进程返回用户空间时处理信号过程中被冻结,因为若其在内核态执行时被冻结，若其正好持有一些锁，则可能会导致死锁// 如果是内核线程，直接冻结，并将状态设置为TASK_INTERRUPTIBLEif (!(p->flags & PF_KTHREAD))fake_signal_wake_up(p);elsewake_up_state(p, TASK_INTERRUPTIBLE);spin_unlock_irqrestore(&freezer_lock, flags);return true;
}// kernel/kernel/signal.coid signal_wake_up_state(struct task_struct *t, unsigned int state)
{// 设置TIF_SIGPENDING信号，在get_signal函数中获取处理set_tsk_thread_flag(t, TIF_SIGPENDING);/** TASK_WAKEKILL also means wake it up in the stopped/traced/killable* case. We don't check t->state here because there is a race with it* executing another processor and just now entering stopped state.* By using wake_up_state, we ensure the process will wake up and* handle its death signal.*/if (!wake_up_state(t, state | TASK_INTERRUPTIBLE))kick_process(t);
}bool get_signal(struct ksignal *ksig)
{struct sighand_struct *sighand = current->sighand;struct signal_struct *signal = current->signal;int signr;if (unlikely(uprobe_deny_signal()))return false;/** Do this once, we can't return to user-mode if freezing() == T.* do_signal_stop() and ptrace_stop() do freezable_schedule() and* thus do not need another check after return.*/try_to_freeze();......} static inline bool try_to_freeze(void)
{if (!(current->flags & PF_NOFREEZE))debug_check_no_locks_held();return try_to_freeze_unsafe();
}static inline bool try_to_freeze_unsafe(void)
{might_sleep();if (likely(!freezing(current)))return false;return __refrigerator(false);
}2.1.2 冻结内核线程
int freeze_kernel_threads(void)
{int error;pr_info("%s:Freezing remaining freezable tasks ... ", STR_KERNEL_LOG_ENTER);pm_nosig_freezing = true;// 传入false表示内核线程，代码流程同2.1.1中try_to_freeze_taskserror = try_to_freeze_tasks(false);if (!error)pr_cont("done.");pr_cont("\n");BUG_ON(in_atomic());if (error)thaw_kernel_threads();return error;
}最后会调用kthread_freezable_should_stop函数执行内线线程冻结：
bool kthread_freezable_should_stop(bool *was_frozen)
{…if (unlikely(freezing(current)))frozen = __refrigerator(true);if (was_frozen)*was_frozen = frozen;return kthread_should_stop();
}2.1.3 小结
进程被冻结主要做了以下事情：
1）设置task状态为TASK_UNINTERRUPTIBLE，表示不能加入就绪队列被调度
2）设置task的flag为PF_FROZEN，表示进程已被冻结
3）调用schedule函数，将task从cpu上调度出来，不让其执行cpu，将寄存器堆栈信息保存到thread_info->cpu_context中
4）进程被解冻时，重新被调度，退出for循环，继续往下执行，重新设置task的状态为TASK_RUNNING
bool __refrigerator(bool check_kthr_stop)
{/* Hmm, should we be allowed to suspend when there are realtimeprocesses around? */bool was_frozen = false;long save = current->state;pr_debug("%s entered refrigerator\n", current->comm);for (;;) {// 设置当前task状态为TASK_UNINTERRUPTIBLEset_current_state(TASK_UNINTERRUPTIBLE);spin_lock_irq(&freezer_lock);// 设置当前task的flag为PF_FROZEN，表示已冻结current->flags |= PF_FROZEN;if (!freezing(current) ||(check_kthr_stop && kthread_should_stop()))current->flags &= ~PF_FROZEN;trace_android_rvh_refrigerator(pm_nosig_freezing);spin_unlock_irq(&freezer_lock);if (!(current->flags & PF_FROZEN))break;was_frozen = true;// 将task从cpu上调度出来，不让其执行cpu，执行schedule函数，会将寄存器堆栈信息保存到thread_info->cpu_context中// task的上下文保存后，停留在该处，下次被唤醒时，重新被调度，退出for循环，往下执行schedule();}pr_debug("%s left refrigerator\n", current->comm);/** Restore saved task state before returning. The mb'd version* needs to be used; otherwise, it might silently break* synchronization which depends on ordered task state change.*/// 被唤醒时，重新设置task的状态set_current_state(save);return was_frozen;
}
EXPORT_SYMBOL(__refrigerator);2.2 进程解冻或唤醒
进程解冻会调用调度模块进行进程唤醒，状态设置为runnable或running.
void __thaw_task(struct task_struct *p)
{unsigned long flags;const struct cpumask *mask = task_cpu_possible_mask(p);spin_lock_irqsave(&freezer_lock, flags);/** Wake up frozen tasks. On asymmetric systems where tasks cannot* run on all CPUs, ttwu() may have deferred a wakeup generated* before thaw_secondary_cpus() had completed so we generate* additional wakeups here for tasks in the PF_FREEZER_SKIP state.*/if (frozen(p) || (frozen_or_skipped(p) && mask != cpu_possible_mask))// 调用调度模块唤醒进程wake_up_process(p);spin_unlock_irqrestore(&freezer_lock, flags);
}线程入队操作并标记线程p为runnable状态，线程标记为TASK_RUNNING，并执行唤醒抢占操作。
int wake_up_process(struct task_struct *p)
{WARN_ON(task_is_stopped_or_traced(p));return try_to_wake_up(p, TASK_NORMAL, 0);
}static int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
{unsigned long flags;int cpu, success = 0;/** If we are going to wake up a thread waiting for CONDITION we* need to ensure that CONDITION=1 done by the caller can not be* reordered with p->state check below. This pairs with mb() in* set_current_state() the waiting thread does.*/smp_mb__before_spinlock();raw_spin_lock_irqsave(&p->pi_lock, flags);if (!(p->state & state))goto out;success = 1; /* we're going to change ->state */cpu = task_cpu(p);/* 使用内存屏障保证p->on_rq的数值是最新的。如果线程已经在运行队列rq里面了，即进程已经处于runnable/running状态。ttwu_remote目的是由于线程 p已经在运行队列rq里面了，并且没有完全取消调度，再次唤醒的话，需要将线程的状态翻转：将状态设置为TASK_RUNNING，这样线程就一直在运行队列里面了。这种情况则直接退出后续流程，并对调度状态/数据进行统计 */if (p->on_rq && ttwu_remote(p, wake_flags))goto stat;#ifdef CONFIG_SMP/* 等待在其他cpu上的线程调度完成 */while (p->on_cpu)cpu_relax();/** Pairs with the smp_wmb() in finish_lock_switch().*/smp_rmb();p->sched_contributes_to_load = !!task_contributes_to_load(p);p->state = TASK_WAKING;/* 根据进程的所属的调度类调用相应的回调函数 */if (p->sched_class->task_waking)p->sched_class->task_waking(p);/* 根据线程p相关参数和系统状态，为线程p选择合适的cpu */cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);/* 如果选择的cpu与线程p当前所在的cpu不相同，则将线程的wake_flags设置为需要迁移，然后将线程p迁移到cpu上 */if (task_cpu(p) != cpu) {wake_flags |= WF_MIGRATED;set_task_cpu(p, cpu);}
#endif /* CONFIG_SMP *//* 线程p入队操作并标记线程p为runnable状态，同时唤醒抢占 */ttwu_queue(p, cpu);
stat:/* 与调度相关的统计 */ttwu_stat(p, cpu, wake_flags);
out:raw_spin_unlock_irqrestore(&p->pi_lock, flags);return success;
}static void ttwu_queue(struct task_struct *p, int cpu)
{struct rq *rq = cpu_rq(cpu);#if defined(CONFIG_SMP)if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {sched_clock_cpu(cpu); /* sync clocks x-cpu */ttwu_queue_remote(p, cpu);return;}
#endifraw_spin_lock(&rq->lock);ttwu_do_activate(rq, p, 0);raw_spin_unlock(&rq->lock);
}static void ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
{
#ifdef CONFIG_SMPif (p->sched_contributes_to_load)rq->nr_uninterruptible--;
#endif//将线程p加入运行队列rq中ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);//将任务标记为可运行的，并执行唤醒抢占。ttwu_do_wakeup(rq, p, wake_flags);
}static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
{activate_task(rq, p, en_flags);p->on_rq = TASK_ON_RQ_QUEUED;/* if a worker is waking up, notify workqueue */if (p->flags & PF_WQ_WORKER)wq_worker_waking_up(p, cpu_of(rq));
}static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
{update_rq_clock(rq);sched_info_queued(rq, p);p->sched_class->enqueue_task(rq, p, flags);
}void activate_task(struct rq *rq, struct task_struct *p, int flags)
{if (task_contributes_to_load(p))rq->nr_uninterruptible--;enqueue_task(rq, p, flags);
}//将任务标记为可运行的，并执行唤醒抢占操作
static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
{check_preempt_curr(rq, p, wake_flags);trace_sched_wakeup(p, true);//将线程p的状态设置为TASK_RUNNINGp->state = TASK_RUNNING;
#ifdef CONFIG_SMPif (p->sched_class->task_woken)p->sched_class->task_woken(rq, p);if (rq->idle_stamp) {u64 delta = rq_clock(rq) - rq->idle_stamp;u64 max = 2*rq->max_idle_balance_cost;update_avg(&rq->avg_idle, delta);if (rq->avg_idle > max)rq->avg_idle = max;rq->idle_stamp = 0;}
#endif
}/*在增加nr_running之前调用enqueue_task（）函数。在这里，将更新公平调度统计数据，然后将线程p的调度实体放入rbtree红黑树中*/
static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
{struct cfs_rq *cfs_rq;struct sched_entity *se = &p->se;for_each_sched_entity(se) {if (se->on_rq)break;cfs_rq = cfs_rq_of(se);// 调度实体存入就绪队列enqueue_entity(cfs_rq, se, flags);/** end evaluation on encountering a throttled cfs_rq** note: in the case of encountering a throttled cfs_rq we will* post the final h_nr_running increment below.*/if (cfs_rq_throttled(cfs_rq))break;cfs_rq->h_nr_running++;flags = ENQUEUE_WAKEUP;}for_each_sched_entity(se) {cfs_rq = cfs_rq_of(se);cfs_rq->h_nr_running++;if (cfs_rq_throttled(cfs_rq))break;// 更新cfs队列权重update_cfs_shares(cfs_rq);// 更新调度实体的平均负载update_entity_load_avg(se, 1);}if (!se) {update_rq_runnable_avg(rq, rq->nr_running);add_nr_running(rq, 1);}hrtick_update(rq);
}
