Source code for torch.distributed.elastic.rendezvous.dynamic_rendezvous
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
import inspect
import logging
import os
import pickle
import socket
import threading
import time
import weakref
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, cast
from torch.distributed import PrefixStore, Store
from torch.distributed.elastic.events import (
NodeState,
construct_and_record_rdzv_event,
)
from .api import (
RendezvousClosedError,
RendezvousError,
RendezvousHandler,
RendezvousParameters,
RendezvousStateError,
RendezvousTimeoutError,
)
from .utils import _delay, _PeriodicTimer
__all__ = ['RendezvousBackend', 'RendezvousTimeout', 'RendezvousSettings', 'DynamicRendezvousHandler', 'create_handler']
log = logging.getLogger(__name__)
def get_method_name(depth=2):
if len(inspect.stack()) > depth:
return inspect.stack()[depth].function
return "no_method_name"
Token = Any
"""Represents an opaque fencing token used by the rendezvous backend."""
[docs]class RendezvousBackend(ABC):
"""Represents a backend that holds the rendezvous state."""
@property
@abstractmethod
def name(self) -> str:
"""Gets the name of the backend."""
[docs] @abstractmethod
def get_state(self) -> Optional[Tuple[bytes, Token]]:
"""Gets the rendezvous state.
Returns:
A tuple of the encoded rendezvous state and its fencing token or
``None`` if no state is found in the backend.
Raises:
RendezvousConnectionError:
The connection to the backend has failed.
RendezvousStateError:
The rendezvous state is corrupt.
"""
[docs] @abstractmethod
def set_state(
self, state: bytes, token: Optional[Token] = None
) -> Optional[Tuple[bytes, Token, bool]]:
"""Sets the rendezvous state.
The new rendezvous state is set conditionally:
- If the specified ``token`` matches the fencing token stored in the
backend, the state will be updated. The new state will be returned
to the caller along with its fencing token.
- If the specified ``token`` does not match the fencing token stored
in the backend, the state won't be updated; instead the existing
state along with its fencing token will be returned to the caller.
- If the specified ``token`` is ``None``, the new state will be set
only if there is no existing state in the backend. Either the new
state or the existing state along with its fencing token will be
returned to the caller.
Args:
state:
The encoded rendezvous state.
token:
An optional fencing token that was retrieved by a previous call
to :py:meth:`get_state` or ``set_state()``.
Returns:
A tuple of the serialized rendezvous state, its fencing token, and
a boolean value indicating whether our set attempt succeeded.
Raises:
RendezvousConnectionError:
The connection to the backend has failed.
RendezvousStateError:
The rendezvous state is corrupt.
"""
[docs]class RendezvousTimeout:
"""Holds the timeout configuration of a rendezvous.
Args:
join:
The time within which the rendezvous is expected to complete.
last_call:
An additional wait amount before completing the rendezvous once the
rendezvous has the minimum number of required participants.
close:
The time within which the rendezvous is expected to close after a
call to :py:meth:`RendezvousHandler.set_closed` or
:py:meth:`RendezvousHandler.shutdown`.
keep_alive:
The time within which a keep-alive heartbeat is expected to
complete.
"""
_ZERO = timedelta(0)
_DEFAULT_TIMEOUTS = {
"join": timedelta(seconds=600),
"last_call": timedelta(seconds=30),
"close": timedelta(seconds=30),
"heartbeat": timedelta(seconds=5),
}
_join: timedelta
_last_call: timedelta
_close: timedelta
_heartbeat: timedelta
def __init__(
self,
join: Optional[timedelta] = None,
last_call: Optional[timedelta] = None,
close: Optional[timedelta] = None,
heartbeat: Optional[timedelta] = None,
) -> None:
self._set_timeouts(join=join, last_call=last_call, close=close, heartbeat=heartbeat)
@property
def join(self) -> timedelta:
"""Gets the join timeout."""
return self._join
@property
def last_call(self) -> timedelta:
"""Gets the last call timeout."""
return self._last_call
@property
def close(self) -> timedelta:
"""Gets the close timeout."""
return self._close
@property
def heartbeat(self) -> timedelta:
"""Gets the keep-alive heartbeat timeout."""
return self._heartbeat
def _set_timeouts(self, **timeouts: Optional[timedelta]):
for name, timeout in timeouts.items():
if timeout is None:
timeout = self._DEFAULT_TIMEOUTS[name]
if timeout <= self._ZERO:
raise ValueError(f"The {name} timeout ({timeout}) must be positive.")
setattr(self, "_" + name, timeout)
@dataclass(repr=False, eq=False, frozen=True)
class RendezvousSettings:
"""Holds the settings of the rendezvous.
Attributes:
run_id:
The run id of the rendezvous.
min_nodes:
The minimum number of nodes to admit to the rendezvous.
max_nodes:
The maximum number of nodes to admit to the rendezvous.
timeout:
The timeout configuration of the rendezvous.
keep_alive_interval:
The amount of time a node waits before sending a heartbeat to keep
it alive in the rendezvous.
keep_alive_max_attempt:
The maximum number of failed heartbeat attempts after which a node
is considered dead.
"""
run_id: str
min_nodes: int
max_nodes: int
timeout: RendezvousTimeout
keep_alive_interval: timedelta
keep_alive_max_attempt: int
@dataclass(eq=True, order=True, frozen=True)
class _NodeDesc:
"""Describes a node in the rendezvous.
Attributes:
addr:
The FQDN of the node or user specified local node address.
pid:
The id of the process in which the rendezvous handler runs.
local_id:
A process-wide unique id.
"""
addr: str
pid: int
local_id: int
def __repr__(self) -> str:
return f"{self.addr}_{self.pid}_{self.local_id}"
class _NodeDescGenerator:
"""Generates node descriptors.
A node descriptor is a combination of an FQDN, a process id, and an auto-
incremented integer that uniquely identifies a node in the rendezvous.
"""
_lock: threading.Lock
_local_id: int
def __init__(self) -> None:
self._lock = threading.Lock()
# An integer that is incremented with each call to generate().
self._local_id = 0
def generate(self, local_addr: Optional[str] = None) -> _NodeDesc:
# This method can be called by multiple threads concurrently; therefore,
# we must increment the integer atomically.
with self._lock:
local_id = self._local_id
self._local_id += 1
return _NodeDesc(local_addr or socket.getfqdn(), os.getpid(), local_id)
class _RendezvousState:
"""Holds the state of a rendezvous.
Attributes:
round:
The current round of the rendezvous.
complete:
A boolean value indicating whether the current round of the
rendezvous is complete.
deadline:
The time at which the current round of the rendezvous will be
considered complete if it is still waiting for nodes to join.
closed:
A boolean value indicating whether the rendezvous is closed.
participants:
A dictionary of the participants and their corresponding ranks.
wait_list:
A set of nodes that are waiting to participate in the next round of
the rendezvous.
last_heartbeats:
A dictionary containing each node's last heartbeat time.
"""
round: int
complete: bool
deadline: Optional[datetime]
closed: bool
participants: Dict[_NodeDesc, int]
wait_list: Set[_NodeDesc]
last_heartbeats: Dict[_NodeDesc, datetime]
def __init__(self) -> None:
self.round = 0
self.complete = False
self.deadline = None
self.closed = False
self.participants = {}
self.wait_list = set()
self.last_heartbeats = {}
def _remove_participant_epilogue(state: _RendezvousState, settings: RendezvousSettings) -> None:
if state.complete:
# If we do not have any participants left, move to the next round.
if not state.participants:
state.complete = False
state.round += 1
else:
if len(state.participants) < settings.min_nodes:
state.deadline = None
class _RendezvousStateHolder(ABC):
"""Holds the shared rendezvous state synced with other nodes."""
@property
@abstractmethod
def state(self) -> _RendezvousState:
"""Gets the local state."""
@abstractmethod
def sync(self) -> Optional[bool]:
"""Reads or writes the latest state.
Returns:
A boolean value indicating whether the local state, in case marked
as dirty, was successfully synced with other nodes.
"""
@abstractmethod
def mark_dirty(self) -> None:
"""Marks the local state as dirty."""
class _BackendRendezvousStateHolder(_RendezvousStateHolder):
"""Holds the rendezvous state synced with other nodes via a backend.
Args:
backend:
The rendezvous backend to use.
settings:
The rendezvous settings.
cache_duration:
The amount of time, in seconds, to cache the last rendezvous state
before requesting it from the backend again.
"""
_backend: RendezvousBackend
_state: _RendezvousState
_settings: RendezvousSettings
_cache_duration: int
_token: Token
_dirty: bool
_last_sync_time: float
_dead_nodes: List[_NodeDesc]
def __init__(
self,
backend: RendezvousBackend,
settings: RendezvousSettings,
cache_duration: int = 1,
) -> None:
self._backend = backend
self._state = _RendezvousState()
self._settings = settings
self._cache_duration = cache_duration
self._token = None
self._dirty = False
self._last_sync_time = -1
self._dead_nodes = []
def _record(self, message: str, node_state: NodeState = NodeState.RUNNING):
construct_and_record_rdzv_event(
name=f"{self.__class__.__name__}.{get_method_name()}",
run_id=self._settings.run_id,
message=message,
node_state=node_state,
)
@property
def state(self) -> _RendezvousState:
"""See base class."""
return self._state
def sync(self) -> Optional[bool]:
"""See base class."""
state_bits: Optional[bytes] = None
token = None
has_set: Optional[bool]
if self._dirty:
has_set = False
state_bits = pickle.dumps(self._state)
set_response = self._backend.set_state(state_bits, self._token)
if set_response is not None:
state_bits, token, has_set = set_response
else:
has_set = None
if self._cache_duration > 0:
# Avoid overloading the backend if we are asked to retrieve the
# state repeatedly. Try to serve the cached state.
if self._last_sync_time >= max(time.monotonic() - self._cache_duration, 0):
return None
get_response = self._backend.get_state()
if get_response is not None:
state_bits, token = get_response
if state_bits is not None:
try:
self._state = pickle.loads(state_bits)
except pickle.PickleError as exc:
raise RendezvousStateError(
"The rendezvous state is corrupt. See inner exception for details."
) from exc
else:
self._state = _RendezvousState()
if has_set and self._dead_nodes and log.isEnabledFor(logging.DEBUG):
node_list = ", ".join(f"'{dead_node}'" for dead_node in self._dead_nodes)
msg = (
f"As part of the sync operation the node(s) {node_list} have been removed from the "
f"rendezvous '{self._settings.run_id}' since they had no heartbeat."
)
self._record(message=msg)
log.debug(msg)
self._token = token
self._dirty = False
self._last_sync_time = time.monotonic()
self._sanitize()
return has_set
def _sanitize(self) -> None:
state = self._state
expire_time = datetime.utcnow() - (
self._settings.keep_alive_interval * self._settings.keep_alive_max_attempt
)
# Filter out the dead nodes.
self._dead_nodes = [
node
for node, last_heartbeat in state.last_heartbeats.items()
if last_heartbeat < expire_time
]
participant_removed = False
for dead_node in self._dead_nodes:
del state.last_heartbeats[dead_node]
try:
del state.participants[dead_node]
participant_removed = True
except KeyError:
pass
try:
state.wait_list.remove(dead_node)
except KeyError:
pass
if participant_removed:
# Common epilogue shared with the _remove_from_participants()
# function of _DistributedRendezvousOpExecutor.
_remove_participant_epilogue(state, self._settings)
def mark_dirty(self) -> None:
"""See base class.
If the local rendezvous state is dirty, the next sync call will try to
write the changes back to the backend. However this attempt might fail
if another node, which had the same state, also made changes and wrote
them before us.
"""
self._dirty = True
class _Action(Enum):
"""Specifies the possible actions based on the state of the rendezvous."""
KEEP_ALIVE = 1
ADD_TO_PARTICIPANTS = 2
ADD_TO_WAIT_LIST = 3
REMOVE_FROM_PARTICIPANTS = 4
REMOVE_FROM_WAIT_LIST = 5
MARK_RENDEZVOUS_COMPLETE = 6
MARK_RENDEZVOUS_CLOSED = 7
SYNC = 8
ERROR_CLOSED = 9
ERROR_TIMEOUT = 10
FINISH = 11
class _RendezvousContext:
"""Holds the context of the rendezvous.
Attributes:
node:
The node descriptor associated with the current rendezvous handler
instance.
state:
The current state of the rendezvous.
settings:
The rendezvous settings.
"""
node: _NodeDesc
state: _RendezvousState
settings: RendezvousSettings
def __init__(
self, node: _NodeDesc, state: _RendezvousState, settings: RendezvousSettings
) -> None:
self.node = node
self.state = state
self.settings = settings
class _RendezvousOpExecutor(ABC):
"""Executes rendezvous operations."""
@abstractmethod
def run(
self,
state_handler: Callable[[_RendezvousContext, float], _Action],
deadline: float,
) -> None:
"""Executes a rendezvous operation.
An operation is run inside a state machine and is expected to transition
the rendezvous from one state to another.
Args:
state_handler:
A callable that is expected to return the next state transition
action based on the current state of the rendezvous.
deadline:
The time, in seconds, at which the operation will be considered
timed-out.
"""
class _DistributedRendezvousOpExecutor(_RendezvousOpExecutor):
"""Executes rendezvous operations using a shared state.
Args:
node:
The node descriptor associated with the current rendezvous handler
instance.
state_holder:
The ``RendezvousStateHolder`` to use to sync the rendezvous state
with other nodes.
settings:
The rendezvous settings.
"""
_node: _NodeDesc
_state: _RendezvousState
_state_holder: _RendezvousStateHolder
_settings: RendezvousSettings
def __init__(
self,
node: _NodeDesc,
state_holder: _RendezvousStateHolder,
settings: RendezvousSettings,
) -> None:
self._node = node
self._state_holder = state_holder
self._settings = settings
def _record(self, message: str, node_state: NodeState = NodeState.RUNNING) -> None:
construct_and_record_rdzv_event(
name=f"{self.__class__.__name__}.{get_method_name()}",
run_id=self._settings.run_id,
message=message,
node_state=node_state,
hostname=self._node.addr,
pid=self._node.pid,
local_id=self._node.local_id,
)
def run(
self,
state_handler: Callable[[_RendezvousContext, float], _Action],
deadline: float,
) -> None:
"""See base class."""
action = None
while action != _Action.FINISH:
# Reads or writes the latest rendezvous state shared by all nodes in
# the rendezvous. Note that our local changes might get overridden
# by another node if that node synced its changes before us.
has_set = self._state_holder.sync()
if has_set is not None:
if has_set:
msg = (
f"The node '{self._node}' has successfully synced its local changes with "
f"other nodes in the rendezvous '{self._settings.run_id}'."
)
else:
msg = (
f"The node '{self._node}' has a stale state and failed to sync its local "
f"changes with other nodes in the rendezvous '{self._settings.run_id}'."
)
self._record(message=msg)
log.debug(msg)
self._state = self._state_holder.state
ctx = _RendezvousContext(self._node, self._state, self._settings)
# Determine the next action to take based on the current state of
# the rendezvous.
action = state_handler(ctx, deadline)
if action == _Action.FINISH:
continue
if action == _Action.ERROR_CLOSED:
raise RendezvousClosedError()
if action == _Action.ERROR_TIMEOUT:
raise RendezvousTimeoutError()
if action == _Action.SYNC:
# Delay the execution by one second to avoid overloading the
# backend if we are asked to poll for state changes.
_delay(seconds=1)
else:
if action == _Action.KEEP_ALIVE:
self._keep_alive()
elif action == _Action.ADD_TO_PARTICIPANTS:
self._add_to_participants()
elif action == _Action.ADD_TO_WAIT_LIST:
self._add_to_wait_list()
elif action == _Action.REMOVE_FROM_PARTICIPANTS:
self._remove_from_participants()
elif action == _Action.REMOVE_FROM_WAIT_LIST:
self._remove_from_wait_list()
elif action == _Action.MARK_RENDEZVOUS_COMPLETE:
self._mark_rendezvous_complete()
elif action == _Action.MARK_RENDEZVOUS_CLOSED:
self._mark_rendezvous_closed()
# Attempt to sync our changes back to other nodes.
self._state_holder.mark_dirty()
def _keep_alive(self) -> None:
msg = (
f"The node '{self._node}' updated its keep-alive heartbeat time for the rendezvous "
f"'{self._settings.run_id}'. Pending sync."
)
self._record(message=msg)
log.debug(msg)
self._state.last_heartbeats[self._node] = datetime.utcnow()
def _add_to_participants(self) -> None:
msg = (
f"The node '{self._node}' added itself to the participants of round "
f"{self._state.round} of the rendezvous '{self._settings.run_id}'. Pending sync."
)
self._record(message=msg)
log.debug(msg)
state = self._state
try:
state.wait_list.remove(self._node)
except KeyError:
pass
# The ranks of the participants will be set once the rendezvous is
# complete.
state.participants[self._node] = 0
self._keep_alive()
if len(state.participants) == self._settings.min_nodes:
state.deadline = datetime.utcnow() + self._settings.timeout.last_call
if len(state.participants) == self._settings.max_nodes:
self._mark_rendezvous_complete()
def _add_to_wait_list(self) -> None:
msg = (
f"The node '{self._node}' added itself to the wait list of round "
f"{self._state.round + 1} of the rendezvous '{self._settings.run_id}'. Pending sync."
)
self._record(message=msg)
log.debug(msg)
self._state.wait_list.add(self._node)
self._keep_alive()
def _remove_from_participants(self) -> None:
msg = (
f"The node '{self._node}' removed itself from the participants of round "
f"{self._state.round} of the rendezvous '{self._settings.run_id}'. Pending sync."
)
self._record(message=msg)
log.debug(msg)
state = self._state
del state.participants[self._node]
del state.last_heartbeats[self._node]
# Common epilogue shared with the sanitizer() function of
# _BackendRendezvousStateHolder.
_remove_participant_epilogue(state, self._settings)
def _remove_from_wait_list(self) -> None:
msg = (
f"The node '{self._node}' removed itself from the wait list of round "
f"{self._state.round + 1} of the rendezvous '{self._settings.run_id}'. Pending sync."
)
self._record(message=msg)
log.debug(msg)
self._state.wait_list.remove(self._node)
del self._state.last_heartbeats[self._node]
def _mark_rendezvous_complete(self) -> None:
msg = (
f"The node '{self._node}' marked round {self._state.round} of the rendezvous "
f"'{self._settings.run_id}' as complete. Pending sync."
)
self._record(message=msg, node_state=NodeState.SUCCEEDED)
log.debug(msg)
state = self._state
state.complete = True
state.deadline = None
# Assign the ranks.
for rank, node in enumerate(sorted(state.participants)):
state.participants[node] = rank
def _mark_rendezvous_closed(self) -> None:
msg = (
f"The node '{self._node}' marked the rendezvous '{self._settings.run_id}' as closed. "
"Pending sync."
)
self._record(message=msg, node_state=NodeState.SUCCEEDED)
log.debug(msg)
self._state.closed = True
def _should_keep_alive(ctx: _RendezvousContext) -> bool:
"""Determines whether a keep-alive heartbeat should be sent."""
try:
last_heartbeat = ctx.state.last_heartbeats[ctx.node]
except KeyError:
return False
return last_heartbeat <= datetime.utcnow() - ctx.settings.keep_alive_interval
class _RendezvousExitOp:
"""Represents a rendezvous exit operation."""
def __call__(self, ctx: _RendezvousContext, deadline: float) -> _Action:
if ctx.node in ctx.state.participants:
if time.monotonic() > deadline:
return _Action.ERROR_TIMEOUT
return _Action.REMOVE_FROM_PARTICIPANTS
return _Action.FINISH
class _RendezvousJoinOp:
"""Represents a rendezvous join operation."""
def __call__(self, ctx: _RendezvousContext, deadline: float) -> _Action:
state = ctx.state
# A closed rendezvous means that it no longer accepts new nodes.
if state.closed:
return _Action.ERROR_CLOSED
is_participant = ctx.node in state.participants
# If we are part of the rendezvous and it is already complete there is
# no further action to take.
if state.complete and is_participant:
return _Action.FINISH
now = time.monotonic()
if now > deadline:
rollback_period = 5 # 5 seconds
# If we still have time to rollback (a short period on top of the
# operation deadline), try to remove ourself from the rendezvous.
# It is okay if we can't though as our keep-alive will eventually
# expire.
if now <= deadline + rollback_period:
# If we are part of the rendezvous, it means we couldn't find
# enough participants to complete it on time.
if is_participant:
return _Action.REMOVE_FROM_PARTICIPANTS
# If we are in the wait list, it means we couldn't wait till the
# next round of the rendezvous.
if ctx.node in state.wait_list:
return _Action.REMOVE_FROM_WAIT_LIST
return _Action.ERROR_TIMEOUT
if state.complete:
# If we are here, it means we are not part of the rendezvous. In
# case the rendezvous has capacity for additional participants add
# ourself to the wait list for the next round.
if len(state.participants) < ctx.settings.max_nodes:
if ctx.node not in state.wait_list:
return _Action.ADD_TO_WAIT_LIST
elif is_participant:
# If the rendezvous has enough number of participants including us,
# check whether we have passed the rendezvous deadline. If yes,
# complete it.
if len(state.participants) >= ctx.settings.min_nodes:
if cast(datetime, state.deadline) < datetime.utcnow():
return _Action.MARK_RENDEZVOUS_COMPLETE
else:
# The rendezvous is not complete yet and we are not part of it. Try
# to join.
return _Action.ADD_TO_PARTICIPANTS
if _should_keep_alive(ctx):
return _Action.KEEP_ALIVE
# At this point either the rendezvous is not complete, but we are part
# of it, which means we have to wait for other participants to join; or
# the rendezvous is complete, but we are not part of it, which means we
# have to wait for the next round.
return _Action.SYNC
class _RendezvousCloseOp:
"""Represents a rendezvous close operation."""
def __call__(self, ctx: _RendezvousContext, deadline: float) -> _Action:
if ctx.state.closed:
return _Action.FINISH
if time.monotonic() > deadline:
return _Action.ERROR_TIMEOUT
return _Action.MARK_RENDEZVOUS_CLOSED
class _RendezvousKeepAliveOp:
"""Represents a rendezvous keep-alive update operation."""
def __call__(self, ctx: _RendezvousContext, deadline: float) -> _Action:
if _should_keep_alive(ctx):
if time.monotonic() > deadline:
return _Action.ERROR_TIMEOUT
return _Action.KEEP_ALIVE
return _Action.FINISH
[docs]class DynamicRendezvousHandler(RendezvousHandler):
"""Represents a handler that sets up a rendezvous among a set of nodes."""
# Static
_node_desc_generator = _NodeDescGenerator()
_this_node: _NodeDesc
_settings: RendezvousSettings
_backend_name: str
_store: Store
_state_holder: _RendezvousStateHolder
_op_executor: _RendezvousOpExecutor
_heartbeat_lock: threading.Lock
_keep_alive_timer: Optional[_PeriodicTimer]
[docs] @classmethod
def from_backend(
cls,
run_id: str,
store: Store,
backend: RendezvousBackend,
min_nodes: int,
max_nodes: int,
local_addr: Optional[str] = None,
timeout: Optional[RendezvousTimeout] = None,
):
"""Creates a new :py:class:`DynamicRendezvousHandler`.
Args:
run_id:
The run id of the rendezvous.
store:
The C10d store to return as part of the rendezvous.
backend:
The backend to use to hold the rendezvous state.
min_nodes:
The minimum number of nodes to admit to the rendezvous.
max_nodes:
The maximum number of nodes to admit to the rendezvous.
local_addr:
The local node address.
timeout:
The timeout configuration of the rendezvous.
"""
# We associate each handler instance with a unique node descriptor.
node = cls._node_desc_generator.generate(local_addr)
settings = RendezvousSettings(
run_id,
min_nodes,
max_nodes,
timeout or RendezvousTimeout(),
keep_alive_interval=timedelta(seconds=5),
keep_alive_max_attempt=3,
)
state_holder = _BackendRendezvousStateHolder(backend, settings)
return cls(node, settings, backend.name, store, state_holder)
def __init__(
self,
node: _NodeDesc,
settings: RendezvousSettings,
backend_name: str,
store: Store,
state_holder: _RendezvousStateHolder,
) -> None:
if not settings.run_id:
raise ValueError("The run id must be a non-empty string.")
if settings.min_nodes < 1:
raise ValueError(
f"The minimum number of nodes ({settings.min_nodes}) must be greater than zero."
)
if settings.max_nodes < settings.min_nodes:
raise ValueError(
f"The maximum number of nodes ({settings.max_nodes}) must be greater than or equal "
f"to the minimum number of nodes ({settings.min_nodes})."
)
self._this_node = node
self._settings = settings
self._backend_name = backend_name
self._store = store
self._state_holder = state_holder
self._op_executor = _DistributedRendezvousOpExecutor(
self._this_node, self._state_holder, self._settings
)
self._heartbeat_lock = threading.Lock()
self._keep_alive_timer = None
def _record(
self,
message: str,
node_state: NodeState = NodeState.RUNNING,
rank: Optional[int] = None,
) -> None:
construct_and_record_rdzv_event(
name=f"{self.__class__.__name__}.{get_method_name()}",
run_id=self._settings.run_id,
message=message,
node_state=node_state,
hostname=self._this_node.addr,
pid=self._this_node.pid,
local_id=self._this_node.local_id,
rank=rank,
)
@property
def settings(self) -> RendezvousSettings:
"""Gets the settings of the rendezvous."""
return self._settings
def get_backend(self) -> str:
"""See base class."""
return self._backend_name
def next_rendezvous(self) -> Tuple[Store, int, int]:
"""See base class."""
msg = (
f"The node '{self._this_node}' attempts to join the next round of the rendezvous "
f"'{self._settings.run_id}'."
)
self._record(message=msg)
log.info(msg)
try:
self._stop_heartbeats()
# Delay the execution for a small random amount of time if this is our
# first run. This will slightly skew the rendezvous attempts across the
# nodes and reduce the load on the backend.
if self._state_holder.state.round == 0:
_delay(seconds=(0, 0.3))
exit_op = _RendezvousExitOp()
join_op = _RendezvousJoinOp()
deadline = self._get_deadline(self._settings.timeout.join)
self._op_executor.run(exit_op, deadline)
self._op_executor.run(join_op, deadline)
self._start_heartbeats()
rank, world_size = self._get_world()
store = self._get_store()
except Exception as e:
self._record(
message=f"{type(e).__name__}: {str(e)}",
node_state=NodeState.FAILED,
)
raise
msg = (
f"The node '{self._this_node}' has joined round {self._state_holder.state.round} of "
f"the rendezvous '{self._settings.run_id}' as rank {rank} in a world of size "
f"{world_size}."
)
self._record(message=msg, rank=rank)
log.info(msg)
return store, rank, world_size
def is_closed(self) -> bool:
"""See base class."""
try:
with self._heartbeat_lock:
self._state_holder.sync()
return self._state_holder.state.closed
except Exception as e:
self._record(
message=f"{type(e).__name__}: {str(e)}",
node_state=NodeState.FAILED,
)
raise
def set_closed(self) -> None:
"""See base class."""
try:
with self._heartbeat_lock:
self._close()
except Exception as e:
self._record(
message=f"{type(e).__name__}: {str(e)}",
node_state=NodeState.FAILED,
)
raise
def num_nodes_waiting(self) -> int:
"""See base class."""
try:
with self._heartbeat_lock:
self._state_holder.sync()
return len(self._state_holder.state.wait_list)
except Exception as e:
self._record(
message=f"{type(e).__name__}: {str(e)}",
node_state=NodeState.FAILED,
)
raise
def get_run_id(self) -> str:
"""See base class."""
return self._settings.run_id
def shutdown(self) -> bool:
"""See base class."""
self._stop_heartbeats()
try:
self._close()
return True
except RendezvousError as ex:
msg = (
f"The node '{self._this_node}' has failed to shutdown the rendezvous "
f"'{self._settings.run_id}' due to an error of type {type(ex).__name__}."
)
self._record(message=msg, node_state=NodeState.FAILED)
log.warning(msg)
return False
except Exception as e:
self._record(
message=f"{type(e).__name__}: {str(e)}",
node_state=NodeState.FAILED,
)
raise
def _close(self) -> None:
op = _RendezvousCloseOp()
deadline = self._get_deadline(self._settings.timeout.close)
self._op_executor.run(op, deadline)
msg = f"The node '{self._this_node}' has closed the rendezvous '{self._settings.run_id}'."
self._record(message=msg, node_state=NodeState.SUCCEEDED)
log.info(msg)
@staticmethod
def _keep_alive_weak(weak_self) -> None:
self = weak_self()
if self is not None:
self._keep_alive()
def _keep_alive(self) -> None:
self._heartbeat_lock.acquire()
op = _RendezvousKeepAliveOp()
deadline = self._get_deadline(self._settings.timeout.heartbeat)
try:
self._op_executor.run(op, deadline)
msg = (
f"The node '{self._this_node}' has sent a keep-alive heartbeat to the rendezvous "
f"'{self._settings.run_id}'."
)
self._record(message=msg)
log.debug(msg)
except RendezvousError as ex:
msg = (
f"The node '{self._this_node}' has failed to send a keep-alive heartbeat to the "
f"rendezvous '{self._settings.run_id}' due to an error of type {type(ex).__name__}."
)
self._record(message=msg, node_state=NodeState.FAILED)
log.warning(msg)
finally:
self._heartbeat_lock.release()
def _start_heartbeats(self) -> None:
self._keep_alive_timer = _PeriodicTimer(
self._settings.keep_alive_interval, self._keep_alive_weak, weakref.ref(self)
)
self._keep_alive_timer.set_name(f"RendezvousKeepAliveTimer_{self._this_node.local_id}")
self._keep_alive_timer.start()
def _stop_heartbeats(self) -> None:
if self._keep_alive_timer is None:
return
self._keep_alive_timer.cancel()
def _get_world(self) -> Tuple[int, int]:
state = self._state_holder.state
return state.participants[self._this_node], len(state.participants)
def _get_store(self) -> Store:
key_prefix = f"torch.rendezvous.{self._settings.run_id}.{self._state_holder.state.round}"
return PrefixStore(key_prefix, self._store)
def _get_deadline(self, timeout: timedelta) -> float:
return time.monotonic() + timeout.total_seconds()
def _get_timeout(params: RendezvousParameters, key: str) -> Optional[timedelta]:
timeout = params.get_as_int(key + "_timeout")
if timeout is None:
return None
return timedelta(seconds=timeout)
[docs]def create_handler(
store: Store, backend: RendezvousBackend, params: RendezvousParameters
) -> DynamicRendezvousHandler:
"""Creates a new :py:class:`DynamicRendezvousHandler` from the specified
parameters.
Args:
store:
The C10d store to return as part of the rendezvous.
backend:
The backend to use to hold the rendezvous state.
+-------------------+------------------------------------------------------+
| Parameter | Description |
+===================+======================================================+
| join_timeout | The total time, in seconds, within which the |
| | rendezvous is expected to complete. Defaults to 600 |
| | seconds. |
+-------------------+------------------------------------------------------+
| last_call_timeout | An additional wait amount, in seconds, before |
| | completing the rendezvous once the minimum number of |
| | nodes has been reached. Defaults to 30 seconds. |
+-------------------+------------------------------------------------------+
| close_timeout | The time, in seconds, within which the rendezvous is |
| | expected to close after a call to |
| | :py:meth:`RendezvousHandler.set_closed` or |
| | :py:meth:`RendezvousHandler.shutdown`. Defaults to |
| | 30 seconds. |
+-------------------+------------------------------------------------------+
"""
try:
timeout = RendezvousTimeout(
_get_timeout(params, "join"),
_get_timeout(params, "last_call"),
_get_timeout(params, "close"),
)
return DynamicRendezvousHandler.from_backend(
params.run_id,
store,
backend,
params.min_nodes,
params.max_nodes,
params.local_addr,
timeout,
)
except Exception as e:
construct_and_record_rdzv_event(
message=f"{type(e).__name__}: {str(e)}",
run_id=params.run_id,
node_state=NodeState.FAILED,
)
raise


Positive Feedback
Negative Feedback
Total Fans
{{vm.totalFansPercent}} % from yesterday
The total number of people who have liked your Page.
The number of your fans who saw any posts on Facebook on a given day
The number of times a Page's profile has been viewed by logged in and logged out people.
The number of stories about your Page's stories, by Page story type
{{vm.range}}


Pat's Salmon Cookie Shops
All locations open daily from 6am to 8pm.
New Location:
minCustomersPerHour:
maxCustomersPerHour:
avgCookiesPerSale:
Add new location


Weather Dashboard
Metric
Imperial
Search
Current
5 day forecast
UVIndex:
UVIndex Legend
UVIndex Legend:
Low
Moderate
High
Very high
extreme


We’ve built a bot to automate the tedious stuff for adding project contributors, so you can focus on your project instead of managing your ReadMe.
Check the Installation doc for how to add it
Check the contribution types in the Emoji Key Cheatsheet
It'll add the Contributor Table for your first time, too


MU Timetable Generator , fully online, for the automatic generation of university timetables .All the operations are online, from data input to final prints, you can use every computer or mobile device connected to the Internet


The system of Hierarchical Event Descriptors (HED) was first developed by Nima Bigdely-Shamlo, during his dissertation work at the Swartz Center at the University of California San Diego (Bigdely-Shamlo et al., 2013), to support a proposed EEG measure search engine. This first-generation HED used a strictly hierarchical term schema (structured vocabulary), making HED event descriptions ('HED strings') each a single term hierarchy (simplified example: shape/square/red).
Within a subsequent U.S. ARL project, Kay Robbins of University of Texas San Antonio pointed out that this would lead to many duplications of terms in the schema tree (a simplified example, the distinct hierarchic strings: Shape/Square/Red; Shape/Triangle/Red). Second-generation HED (Bigdely-Shamlo et al. 2016) therefore made HED strings heterarchical (comma-separated collections of schema hierarchies) -- a simplified example: (Shape/Square, Color/Red), (Shape/Triangle, Color/Red); here the term red needs appear in only one place in the HED schema, in its native (Color) hierarchy.
Most recently, third-generation HED incorporated some key design advances to ease the process of data annotation, improve the efficiency of HED tools for event search, and create a convenient way to marry HED annotation to BIDS event specification (Robbins et al., 2021, Robbins et al., 2022). HED (Gen 3) also includes the concept of extending the HED vocabulary by adding field-specialized HED Library Schema.
HED is now maintained and developed by the volunteer HED Working Group, whose work is currently funded in part by a grant from the US National Institute of Mental Health (NIMH, RF1-MH126700). HED data are housed online in its Github hed-standard site; its public-facing website is www.hedtags.org. HED Tools for performing HED annotation and event search are available online, for use in Python scripts, and for MATLAB, in particular within the EEGLAB environment. HED annotation is now formally accepted for use with data formatted according to the BIDS standards.
To inquire about contributing to further HED development or joining the HED Working Group, follow links on the HEDtags site.
Scott Makeig (Strategic Lead<EMAIL_ADDRESS>Kay Robbins (Technical Lead<EMAIL_ADDRESS>Dung Truong<EMAIL_ADDRESS>Monique Denissen<EMAIL_ADDRESS>Yahya Shirazi<EMAIL_ADDRESS>Deepa Gupta<EMAIL_ADDRESS>Arnaud Delorme<EMAIL_ADDRESS>Dora Hermes<EMAIL_ADDRESS>Tal Pal Attia<EMAIL_ADDRESS>Ian Callanan<EMAIL_ADDRESS>Alexander Jones<EMAIL_ADDRESS>In addition to current support from NIH 1R01MH126700-01, the HED Working Group gratefully acknowledges past support from the Army Research Laboratory under Cooperative Agreement Number W911NF-10–2–0022NIH as well as from NIH R01 EB023297–03, R01 NS047293-l4, and R24 MH120037–01 and the Swartz Foundation.


BarkMatch is a website that allows users to browse dogs and sign up to walk a dog from a list of dogs uploaded by a dog shelter.
Barkmatch was created using Node.js and several node packages. The project used the express node package to create the server, view routes, and api routes. The express handlebars node package was used to render templates created on the server side with data pulled in from the database. The Sequelize node package was used for all of the queries to the MySql database. We used node NodeMailer to create and send a confirmation email to a user once they sign up to walk a dog. Authentication was implemented using the Passport node package. All of the styling was done using the Bulma css framework along with custom css.


Ingredients
- Egg yolk 1
- milk 200ml
- lemon half
- orange juice right amount
- honey right amount
- ice cube right amount
Instructions
- Add the egg yolk to the shake cup and pour the milk.
- Squeeze the lemon juice and add a tablespoon of honey.
- Fill up with ice cubes. Close the lid and shake vigorously
- Pour into the cup then enjoy it~


Message to alert the user to a problem goes here
Optional description of the errors and how to correct them
-
{% if errorMissingTitle %}
- Descriptive link to MISSING IMAGE error {% endif %}
Partner logos
Partner logos will be displayed on the footer of your webpage.
Images will appear in the order that you upload them.


Project examples and code repository for those who are new to html, css and js and want to improve themselves.
Note: All projects belong to me.


{# templates/organization/organization_list.html #} {% extends "template_base.html" %} {% block title %}Organizations{% endblock %} {% block content %} < Back to Admin Home
| {{ organization.organization_name }} | {{ organization.twitter_handle }} | edit |
(no organizations found)
{% endif %} {##} {# Import Organizations#} {#
#} {% endblock %}


My Website: Faults and All
A review of prototype Version 1.1 of benjaminkinzer.com
This site was a further exploration into Prototyping within the web browser and skipping the various Photoshop files or Sketch boards. I love code and with Jekyll and SASS I can generate a fully functional front-end without spending countless hours managing multiple files. This method also provides me with room to experiment and test quickly how responsive the site actually is.
The goal was also to share my insights and maintain a history of my process and to eventually build out an archive of all my work that would function something like a Novel written by Mark Z. Danielewski that includes a CLIFF NOTES version for the people who only like staring at pictures.
Unfortunately I’ve been so busy this has been more like Slack Prototyping.
The Problem
Currently the site is a user experience nightmare for my audience and for myself as far as updating is concerned. So let’s dive in.
Home
For me the home page works well, it’s not ideal. But the feedback I have received is positive. The user has a few jumping points, but the work is there before them to interact with. I think in the future it will be worth considering a redesign, but for now it works.
For Consideration
- Journal Entries on Home
Projects
This page does not exist, so people must click on BENJAMIN KINZER to reach the project thumbnails. This has caused a lot of confusion in navigating between projects. The user also must scroll to the top or use the back button to reach other projects.
For Consideration
- Add Projects to navigation
- ADD MORE WORK!
Project Detail
Project detail from a design perspective is alright and it works well on all screen sizes.  All the images are background images contained in DIVs. This was so that I could deal with the Art Direction of all the images on various screen sizes.
$project-folder: '21stamendment';
$client: '21a';
%grid {
@include col(1/1);
height:205px;
@include min-screen(nth($res-collection, 2)*1px) {
@include col(1/2, $cycle: 2);
height:325px;
}
}
.hell-high-watermelon {
@extend %grid;
@include background-image("#{$project-folder}/#{$client}-mainline-hellorhigh.png");
background-position:center;
background-size:118%;
@include min-screen(nth($res-collection, 2)*1px) {
background-size:auto;
}
}
At the time it was a great idea, but it also required a lot of fussing with background image sizes and managing a stylesheet for each project. I’ve decided to move away from this method and begin implementing
As for the user, well there is a lot of confusion and extra work to navigate between projects. The user probably feels like they were brought back into the stone ages.
Let’s consider a few scenarios
- A User lands on the home page and clicks on a Project Thumbnail. They’re like cool this is fun and scroll down. They now have the option to either use the browser back button or scroll to the top of the page can
Apparently all my project pages have a 100% bounce rate.  If I were viewing my site I would be frustrated that I had to click the back button or scroll to the top to see click on BENJAMIN KINZER to view other work. What if they landed on a specific project page, that rules out the browser back button.
For Consideration
- Previous and Next Project Arrows with Thumbnails
- Possibly having related projects in the footer
- Some kind of menu that allows them to jump to other projects within the site
Profile
So this page from a user stand point causes a lot of confusion. I’ve actually used the copy on the page in conversation and it often leads to blank stares. Not sure that is helping me out very much.
For Consideration
- Write something that actually makes sense to the reader
- Develop a more friendly concept that conveys the BK story
Love
This page is one of my favorites, mainly because it’s people talking about how great I am. But it looks like someone forgot to design the actual page.
The heart icon also appears differently on mobile screens since I’m using an ACSII character. Which means it turns into the default heart emoticon on different mobile operating systems: [Image Heart Icon]
For Consideration
- Swap out with a SVG icon
- Use words instead of icon
Story Behind This Site
I absolutely hate this page. It really should not live within the top level navigation. My thought was to document the process of this site and rapid prototyping with this page, but I think that can be better achieved with a Journal page, like the one you are viewing now.
For Consideration
- Swap out with a SVG icon
- Use words instead of icon
Contact
People haven’t had a difficult time finding how to contact me, so that is a success having the information at the bottom of the site. However I do believe I could make it easier for people to find a way to contact me, besides on the profile or bottom of the page.
Navigation
I had the intention of making the Navigation work more like breadcrumbs. I’m still excited about this idea and will be prototyping this in the next phase.
Bigger Vision
My goal is to create an archive for my work, but also make it easy for anyone to either dive deeper or just stare at pretty pictures. My thinking is to create a site that functions like , but also provide a the ability to just see the Cliff Note version. But I will discuss this more in future posts. For now we will just try to adjust a few items outlined above.


Multiple instances, multiple destinations, separate JS file
C++
C#
Haskell
Java
JavaScript
Lisp
Lua
MATLAB
NewLISP
PHP
Perl
SQL
Unix shell
Undo
Redo
Known languages
C++
C#
I want to learn
SQL
Unix shell
C++
C#
Haskell
Java
JavaScript
Lisp
Lua
MATLAB
NewLISP
PHP
Perl
SQL
Unix shell
Undo
Redo
Known languages
C++
C#
I want to learn
SQL
Unix shell


pymatgen.analysis.magnetism.jahnteller module¶
JahnTeller distortion analysis.
-
class
JahnTellerAnalyzer
[source]¶ Bases:
object
Will attempt to classify if structure may be Jahn-Teller active. Class currently uses datafile of hard-coded common Jahn-Teller active ions. If structure is annotated with magnetic moments, will estimate if structure may be high-spin or low-spin. Class aims for more false-positives than false-negatives.
Init for JahnTellerAnalyzer.
-
get_analysis
(structure: pymatgen.core.structure.Structure, calculate_valences: bool = True, guesstimate_spin: bool = False, op_threshold: float = 0.1) → Dict[source]¶ Convenience method, uses get_analysis_and_structure method.
Obtain an analysis of a given structure and if it may be Jahn-Teller active or not. This is a heuristic, and may give false positives and false negatives (false positives are preferred).
- Parameters
structure – input structure
calculate_valences – whether to attempt to calculate valences or not, structure should have oxidation states to perform analysis (Default value = True)
guesstimate_spin – whether to guesstimate spin state from magnetic moments or not, use with caution (Default value = False)
op_threshold – threshold for order parameter above which to consider site to match an octahedral or tetrahedral motif, since Jahn-Teller structures can often be quite distorted, this threshold is smaller than one might expect
- Returns
analysis of structure, with key ‘strength’ which may be ‘none’, ‘strong’, ‘weak’, or ‘unknown’ (Default value = 0.1)
-
get_analysis_and_structure
(structure: pymatgen.core.structure.Structure, calculate_valences: bool = True, guesstimate_spin: bool = False, op_threshold: float = 0.1) → Tuple[Dict, pymatgen.core.structure.Structure][source]¶ Obtain an analysis of a given structure and if it may be Jahn-Teller active or not. This is a heuristic, and may give false positives and false negatives (false positives are preferred).
- Parameters
structure – input structure
calculate_valences – whether to attempt to calculate valences or not, structure should have oxidation states to perform analysis (Default value = True)
guesstimate_spin – whether to guesstimate spin state from magnetic moments or not, use with caution (Default value = False)
op_threshold – threshold for order parameter above which to consider site to match an octahedral or tetrahedral motif, since Jahn-Teller structures can often be quite distorted, this threshold is smaller than one might expect
- Returns
analysis of structure, with key ‘strength’ which may be ‘none’, ‘strong’, ‘weak’, or ‘unknown’ (Default value = 0.1) and decorated structure
-
get_magnitude_of_effect_from_species
(species: Union[str, pymatgen.core.periodic_table.Specie], spin_state: str, motif: str) → str[source]¶ Get magnitude of Jahn-Teller effect from provided species, spin state and motif.
- Parameters
species – e.g. Fe2+
spin_state – “high” or “low”
motif – “oct” or “tet”
Returns: “none”, “weak” or “strong
-
static
get_magnitude_of_effect_from_spin_config
(motif: str, spin_config: Dict[str, float]) → str[source]¶ Roughly, the magnitude of Jahn-Teller distortion will be: * in octahedral environments, strong if e_g orbitals unevenly occupied but weak if t_2g orbitals unevenly occupied * in tetrahedral environments always weaker
- Parameters
motif – “oct” or “tet”
spin_config – dict of ‘e’ (e_g) and ‘t’ (t2_g) with number of electrons in each state
Returns: “none”, “weak” or “strong”
-
is_jahn_teller_active
(structure: pymatgen.core.structure.Structure, calculate_valences: bool = True, guesstimate_spin: bool = False, op_threshold: float = 0.1) → bool[source]¶ Convenience method, uses get_analysis_and_structure method. Check if a given structure and if it may be Jahn-Teller active or not. This is a heuristic, and may give false positives and false negatives (false positives are preferred).
- Parameters
structure – input structure
calculate_valences – whether to attempt to calculate valences or not, structure should have oxidation states to perform analysis (Default value = True)
guesstimate_spin – whether to guesstimate spin state from magnetic moments or not, use with caution (Default value = False)
op_threshold – threshold for order parameter above which to consider site to match an octahedral or tetrahedral motif, since Jahn-Teller structures can often be quite distorted, this threshold is smaller than one might expect
- Returns
boolean, True if might be Jahn-Teller active, False if not
-
static
mu_so
(species: Union[str, pymatgen.core.periodic_table.Specie], motif: str, spin_state: str) → Optional[float][source]¶ Calculates the spin-only magnetic moment for a given species. Only supports transition metals.
- Parameters
species – Species
motif – “oct” or “tet”
spin_state – “high” or “low”
- Returns
Spin-only magnetic moment in Bohr magnetons or None if species crystal field not defined
-
tag_structure
(structure: pymatgen.core.structure.Structure, calculate_valences: bool = True, guesstimate_spin: bool = False, op_threshold: float = 0.1) → pymatgen.core.structure.Structure[source]¶ Convenience method, uses get_analysis_and_structure method. Add a “possible_jt_active” site property on Structure.
- Parameters
structure – input structure
calculate_valences – whether to attempt to calculate valences or not, structure should have oxidation states to perform analysis (Default value = True)
guesstimate_spin – whether to guesstimate spin state from magnetic moments or not, use with caution (Default value = False)
op_threshold – threshold for order parameter above which to consider site to match an octahedral or tetrahedral motif, since Jahn-Teller structures can often be quite distorted, this threshold is smaller than one might expect
- Returns
Decorated Structure, will be in primitive setting.
-


mor::Clades - A module for the mor pacakge.
use mor::Settings; use mor::DatabaseIO; use Bio::TreeIO; use vars qw(*settings $maindb ); my $cladeTree = $treeIO->next_tree(); $MySQLtable = "table"; mor::Clades::updateClades($MySQLtable, $cladeTree);
This module requires the settings hash and maindb object to be creaeted before it is called.
This package should be called with mor::Clades::updateClades. It will take the constraints table and rip apart the tree passed in to create as many different clades as are in the table.
Title : _getCladeConstraint Usage : Internal method should not be used Function: Grabs a list of accession numbers that are constraints to the given clade name. Returns : Array reference to the list of accession numbers Args : Clade name MySQL table
Title : _getNodeIdHash Usage : Internal method should not be used Function: Takes a Bio::Tree object and returns a hash of accession numbers where the key is the accession number and the value stored is 1 Returns : Hash reference to the list of accession numbers Args : Bio::Tree object
Title : _getSubtree Usage : Internal method should not be used Function: Creates a Bio::Tree object that represents the clade Returns : Either the Bio::Tree clade or 0 if the method failed Args : Array reference of the constraints Bio::Tree object that has the main tree Hash reference to the exisiting accession numbers
Title : updateClades Usage : mor::Clades::updateClades($MySQLtable, $Bio::TreeOBJ) Function: Updates the MySQL table to have newick trees that represent the clade name. Returns : nothing Args : the name of a MySQL table Bio::Tree object with the main tree data
Title : _getTree Usage : Internal method should not be used Function: Gets a tree from the database Returns : Bio::Tree object of the current tree stored in trees Args : a string that is the name of the tree i.e. "PARTree"
Title : updateSingleClade Usage : mor::Clades::updateClades("cladeName") Function: This will update the database with the new clade. Or it will create a clade if it has not been run before. Returns : nothing Args : A string that is the name of the clade to be updated
The mor Team - http://mor.clarku.edu/morTeam.php


Terminal Whispering
Do more than print text with your terminal
Thomas Ballinger | @ballingt |<EMAIL_ADDRESS>|<EMAIL_ADDRESS>Hello! I'd like to talk about how to use our terminals - we use them every day, we might as well learn how. This is a talk not about shell scripting, how to use the command line - but about the actual terminal.
How we use terminals
Many of us use terminals daily - typically in a call and response manner:
The Python interactive interpreter works like this, Bash works like this
But here another program you may be familiar with - it doesn't do call and response.
It takes over the whole screen.
At least 3 secret techniques we want to learn about, but there are a lot encompassed in that last one.
What are terminals?
What are terminals?
Well, we're usually talking about terminal emulators.
When I say "terminal", I'm usually talking about a terminal emulator.
You can see the little mac GUI stuff in this picture -
that's sort of important, we're going to talk about terminals as they exist
in unix systems - I don't know how much of this is translatable to the Windows console.
Windows users: open up a PUTTY terminal on a server for this or use Cygwin
So what do I mean by terminal?
I mean the interface a command line program has to the user - where
an interactive command line program assumes it's output is going
to be displayed, and where it's expected to get input from.
This means I'll be lumping in things the kernal does in tandem with the terminal emulator,
because this is about the interface presented to the application.
What is it that we're emulating?
I usually think we're emulating these things:
video terminals. Applications would be written
anticipating that the human user would be sitting in front
of one of these. But video terminals were emulating something
else originally...
Teleprinters or teletypewriters, or one of the brands...
Things like this have been around for over a hundred years!
Check out a youtube video of this stuff, they're
mechanical marvels!
These had been being used to send telegraphs for a long time
(great 1940 video about how they work: https://www.youtube.com/watch?v=HcMHam54EOI)
I'd like to read you a bit from a teletype ad, this one is from 1957.
 With it, written word can be sent instantaneously by wire...within the office or plant...or clear across the country; to a single destination...or to any number at the same time.
In today's business world, in fact, Teletype equipment is often more than a communications instrument. It is a basic element in production control sytems...its ability to transmit and reproduce text and punched tape is harnessed to office automation...it provides a "conveyor system" for channeling complex raw data to a computing center thousands of miles away.
Whether leased through your telephone company or other communication companies -- or purchased outright from us -- the versatile line of Teletype equipment can be tailored to almost any conceivable demand. For further information, contact your telephone company or other communications company, or send for our free booklet, "The ABC's of Teletype Equipment," Teletype Corporation, Dept. F7, 4100 Fullerton Ave., Chicago 39, Illinois.
1957 Golden Anniversary Year
The way teletypes work lays the foundation for the way
terminal emulators still work today.
(originally 1907)
application--------maybe copper?
Λ |
| |
signal over metal wire |
| |
typewriter<--------------+
Λ |
| ink
| |
| V
| paper
|
fingers pressing keys
A user types at an electronic typewriter which both puts ink to
paper immediately...
application--------maybe copper?
Λ |
| |
signal over metal wire |
| |
teletypewriter<-----------+
Λ |
| ink
| |
| V
| paper
|
fingers pressing keys
and also sends some kind of signal down a wire.
This could be only 5 bits for telegraph stuff
, then ANSI made it 7 -
who knows if you need a whole byte or not? Write a device driver
that deals with whatever telegraph stuff was already in place
application--------maybe copper?
Λ |
| |
signal over metal wire |
| |
teletypewriter<-----------+
Λ |
| ink
| |
| V
| paper
|
fingers pressing keys
The application sends back data that gets printed by the teletype.
Once again, very interesting on YouTube videos! I'll link at the end.
Next, let's look at the modernized version of our diagram...
application-------fwrite(bytes)
Λ |
| |
fread() returns bytes |
| |
terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
| |
keypresses V
| screen
keyboard
"terminal" here means our terminal emulator.
`read` and `write` are system calls - we call them
on special files like standard in and out.
The application makes both of these calls.
application------print()--+
Λ |
| |
input() |
| |
terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
| |
keypresses V
| screen
keyboard
Or as we call them in python, input and print.
This is a simplified diagram - there's probably some stuff between the application and the signals.
If it's so simplified, why do we need this intermediate step? Because keypresses also cause things
to be displayed, and messages that are sent might result in other things than text being displayed.
We'll look at input in a moment, but first let's look at...
What bytes can we write and what happens?
application-------write(bytes)
Λ |
| |
read() returns bytes |
| |
terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
keypresses V
| screen
keyboard
What bytes can we send?
ASCII bytes
sys.stdout.buffer.write(b'some bytes')
control characters ([chr(x) for x in range(32)]
e.g. '\a\n\r\v\b\f\t'
)
ASCII is a mapping of signals to meanings, when the signals getting sent on those metal wires
were sort of standardized in the 60's.
ASCII is a mapping of signals to meanings, when the signals getting sent on those metal wires
were sort of standardized in the 60's.
Even though you only need 5 bits to get all the letters, now we'll splurge and use 7 bits(!)
Once you start using 8-bit bytes to store a character each, we have an extra one - but ASCII was before this.
First we have literals - what you send is what will appear on screen, or in your telegraph message.
Control characters let us do special things... (demo!)
* python session first - but it's confusing to see
the returns from the interactive Python session
* write on a socket
* literals
* newlines
* carriage return
* alert
* linefeed, backspace
We've unlocked the secret of progress bars!
import time, sys
def report_progress(ratio, width=50):
filled = '=' * int(ratio * width)
rest = '-' * (width - int(ratio*width))
sys.stderr.write('\r|'+filled+rest+'|')
sys.stderr.flush()
for i in range(101):
report_progress(i/100.0, 50)
time.sleep(.01)
Notice: flush, writing strings instead of bytes
You might notice that we're not writing to the buffer anymore, and we're not writing bytes
What bytes can we send?
ASCII bytes
sys.stdout.buffer.write(b'some bytes')
control characters ([chr(x) for x in range(32)]
e.g. '\a\n\r\v\b\f\t'
)
terminal probably using an encoding such that several bytes to make a character:
sys.stdout.buffer.write(b'\xc3\xb1')
sys.stdout.write(u'ñ')
(via .encoding
)
Now we're getting to the things that don't make sense with typewriters.
What bytes can we send?
ASCII bytes
sys.stdout.buffer.write(b'some bytes')
control characters ([chr(x) for x in range(32)]
or '\a\n\v\b\f\t'
)
several bytes may encode a single character:
sys.stdout.buffer.write(b'\xc3\xb1')
sys.stdout.write(u'ñ')
(via .encoding
)
ANSI escape sequences ('\x1b[31m'
)
ANSI escape sequences
Sequence
Meaning
\x1b[23A
move cursor 23 rows up
\x1b[2J
clear entire screen
\x1b[ ?25l
hide the cursor
\x1b[1m
start writing in bold
\x1b[31m
start writing in red
See Wikipedia entry on ANSI escape sequences for more.
VT52 allowed cursor placement. In widespread use by the 80s.
The VT100 was a big one. Applications would send these things to the terminal,
so you wanted them to work.
Move cursor: draw windows!

The terminal is put into "red" mode, and everything after that is red until it stops being red. There was state in the typewriters, but it was just "how far to the right is the cursor."
Imagine it's 1984, maybe you're 13 or maybe you're 50. You're excited to use software
or log into bbs systems that do cool things with your video terminal, but they aren't
compatible! Terminal hardware might have had a switch on it to choose which
other terminal to emulate. Need an terminal compatibility layer - a jquery analogue
to smooth over terminal differences.
Compatibility Concerns
termcap, terminfo
curses
tput
infocmp
Terminfo is what we use for this - it's a database of terminals, and it examines the terminal's $TERM value. In the shell you'd use tput...
BUT the ANSI escape sequences are mostly honored, but that's like being optimistic and using cool new JavaScript features even though they won't work on
all browsers. And there are other commands not in the ASCII spec that are like using new ecma script stuff. For browsers, there's a thing called jquery
which acts as a compatibility layer between web application code and browsers.
echo "Input please"
tput cuu1
tput cuu1
tput el
read akey
tput clear
code sample from http://aplawrence.com/Basics/tput_stty.html
Blessings - Erik Rose - see "Designing Poetic APIs" from PyCon 2014
from blessings import Terminal
t = Terminal()
print t.bold('Hi there!')
print t.bold_red_on_bright_green('It hurts my eyes!')
with t.location(0, t.height - 1):
print 'This is at the bottom.'
BUT I wouldn't use ncurses (import curses) in the Python stdlib either. Good as a compatibility layer, bad as an api.
Erik Rose gave a great talk last year called "designing poetic apis" in which he uses his Blessings library as an example.
There are at least two reasons to use jquery: you like the abstractions it gives you, and compatibility.
In my experience, the compatibility is less important because the ASCII standard mostly took hold.
But it's still nicer than hardcoding, and it's definitely. Now that you know these are just bytes you're
still responsible for writing to the terminal, you're prepared to using Blessings!
Let's solve the fullscreen mystery with Blessings
Secrets unlocked by writing bytes to the terminal
Colored and styled text (bold, underline, blink...)
Moving the cursor anywhere
Alternate screen
Hide cursor
And much more (scroll regions, save/restore cursor position; man tput for more)
Bytes you're going to write to the terminal that aren't displayed, but instead change the way the terminal works.
What happens when the user types at the keyboard?
application-------write(bytes)
Λ |
| |
read() returns bytes |
| |
terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
keypresses V
| screen
keyboard
What happens when the user types at the keyboard?
line discipline
data not available to application until the user hits return
backspace, delete word, reprint line, delete line
(try cat to see this behavior)
other terminal settings
Terminal (or something on the wire between the terminal
and the the application) doesn't give the bytes until
the user hits enter.
line editing discipline - before the application gets
the bytes, you can use delete! And delete word!
application--------maybe copper?
Λ |
| |
signal over metal wire |
| |
| |
teletypewriter<-----------+
Λ |
| ink
| |
| V
| paper
|
fingers pressing keys
Echoing back characters - the terminal is doing this,
just like a typewriter! But it doesn't have to.
Modified diagram! We have knobs on the terminal
emulator, and we adjust them with
+----application-------write(bytes)
| Λ |
get | |
& read() returns bytes |
set | |
| | |
+---⚒terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
keypresses V
| screen
|
keyboard
There are knobs and dials on your terminal!
We change this stuff with ioctrl, cbreak, raw, etc
class of system calls for messings with this stuff
Relevant man pages
raw
termios
ioctl
fcntl
stty
isatty
We change this stuff with ioctrl, cbreak, raw, etc
class of system calls for messings with this stuff
I'll show a quick example that your eyes are welcome to glaze over for - because afterwards
I'll tell you to use a library for it.
More important is knowing you can change these things.
Turning off echo, using stty
$ stty -echo
$ echo "echo is turned off"
$ stty echo
Turning off echo in Python
>>> import getpass
>>> getpass.getpass()
Password:
'secret'
Turning off echo in Python
def getpass(prompt="Password: "):
import termios, sys
fd = sys.stdin.fileno()
old = termios.tcgetattr(fd)
new = termios.tcgetattr(fd)
new[3] = new[3] & ~termios.ECHO # lflags
try:
termios.tcsetattr(fd, termios.TCSADRAIN, new)
passwd = raw_input(prompt)
finally:
termios.tcsetattr(fd, termios.TCSADRAIN, old)
return passwd
from https://docs.python.org/2/library/termios.html
Important parts about this:
Don't worry too much about this - the gist is that we're checking the knobs and setting the knobs.
The finally here is really important - so a context manager might be nice!
import sys, termios
class NoEcho:
def __init__(self, in_stream):
self.fd = in_stream.fileno()
def __enter__(self):
self.orig = termios.tcgetattr(self.fd)
new = termios.tcgetattr(self.fd)
new[3] = new[3] & ~termios.ECHO
termios.tcsetattr(fd, termios.TCSADRAIN, new)
def __exit__(self, exc_type, exc_value, traceback):
termios.tcsetattr(self.fd), termios.TCSADRAIN, self.orig)
with NoEcho(sys.stdin):
h = sys.stdin.readline()
print(h)
This is an excellent fit for a context manger, because this is state external to our
Python program - if we crash, we don't want the terminal to be unsuable by the user.
(tip: if it ever is, try `reset`. Twice.)
Secret techniques made possible with these dials
turn off echo
send keys immediately
current terminal size
nonblocking input
os.isatty(sys.stdout.fileno())
Some of these are bucketed up in two collections of dial settings: raw, cbreak
You can do these yourself, I recommend context managers like we saw. Or use a library.
cbreak and raw - bundles of these modes you'll want to use.
Blessings can get terminal size, and once Blessed is merged will be able to do more of these.
Nonblocking is different, it's fcntl - it's a file descriptor thing. But also relevant!
What happens when the user types at the keyboard
buffering, line editing discipline
data not passed to application until the user hits return
backspace, delete word, reprint line, delete line - not much else
to see this, try cat (it doesn't use readline)
echoing back characters
mapping bytes to keys (try cat!)
Example bytes/key mappings
with my particular setup
Bytes read
Key presses
b'a'
a
b'\x1b[23~'
F11
b'\x1b[Z'
Shift-Tab
b'\x1b[1;10B'
Meta-Shift-Down
b'\x1b[1~'
Home
What happens when the user types at the keyboard
buffering, line editing discipline
data not passed to application until the user hits return
backspace, delete word, reprint line, delete line - not much else
to see this, try cat (it doesn't use readline)
echoing back characters
mapping bytes to keys
bytes sent without being typed (cursor query)
Finally for terminal input, not all the bytes you read come from the user!
You should try this one:
>>> print("\x1b[6n")
+---- application-------write(bytes)
| Λ |
get | |
& read() returns bytes |
set | |
| | |
+---⚒ terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
keypresses V
| screen
keyboard
Particularly interesting because writing bytes doesn't do anything
to the display, we just get some informative bytes back the next time
we call read!
Important to do this in unbuffered mode, otherwise they'll just show
up on the terminal line as though the user typed them.
Try it!
>>> print("\x1b[6n")
>>> 1R
>>> print("\x1b[6n")
>>> ESC[3;1R
This doesn't make sense without being in unbuffered mode
How can the terminal notify the application?
+----application-------write(bytes)
| Λ ⚡ |
get | ⚡ |
& read() ⚡signals! |
set | ⚡ |
| | ⚡ |
+---⚒terminal<-----------------+
Λ |
| +--GUI stuff--+
| |
keypresses V
| screen
keyboard
The terminal can push things: window changes, suspend, etc.
The behavior of these is set with our knobs.
Unix signals triggered from the terminal
ctrl-c, suspend, sigwinch
For a given terminal, you can customize when which signals are sent. Cbreak will make some keystrokes trigger signals, raw won't.
These interrupt reads!
PEP 475 - retry system calls failing with EINTR, but now this works differently
This is the difference between cbreak and raw
cooked, raw and cbreak
Cbreak and raw are combinations of settings that are generally useful. Both turn off echo, unbuffer your input. This is how you should get the settings you want.
Fancy terminal tricks in practice
You should now understand *how* the terminal works, but that doesn't mean you need to code this up yourself.
Now before I wrap up I want to give you a brief overview of tools available for doing cool terminal stuff.
Blessings - Erik Rose
from blessings import Terminal
t = Terminal()
print t.bold('Hi there!')
print t.bold_red_on_bright_green('It hurts my eyes!')
with t.location(0, t.height - 1):
print('This is at the bottom.')
You're prepared to use Blessings because you know that it provides strings that you're going to need to write to the terminal.
Erik Rose shoutout!
Blessings frees you from the curses assumption that you want a fullscreen app,
but python prompt toolkit does better for this.
x84, Blessed (soon included in blessings) - Jeff Quast
There will soon be input key handling, width calculations,
Blessed - Jeff Quast
from blessed import Terminal
t = Terminal()
print term.center(term.bold('X'))
with t.cbreak():
print(sys.stdin.read(1))
val = t.inkey(timeout=5)
if val:
print(val)
Urwid
Urwid for big GUIs - Java-like gui stuff
Check out pudb, mitmproxy, a version of bpython
If you're using the full screen mode
Review fullscreen mode! vim, emacs, tmux vs something that doesn't take over the full screen
Curses is often used this way
Click, Clint
Libraries for command line utilities
Still provide colored strings, getting a single character, detect whether stdout is a terminal like Blessings
readline
If you want to do things yourself, great - otherwise there's readline.
Curtsies
More of a framework, less composable
Once you understand the problem, you want somethign less handholdy and more flexible
layout engine
Unique features: (more) composable colored strings, diffing renderer, grids of terminal-width characters
Python Prompt Toolkit
Used for some cool tools, colors - maybe wait for the user to press a key
*NOT* fullscreen, lets you do cool things in a command line utility-like script
pty
this is how terminal emulators work!
pexpect
Thomas Kluyver and Jeff Quast
Python version of expect, for ru
termcast - Jesse Luehrs
$ telnet termcast.org
https://github.com/doy/python-termcast-server
https://github.com/doy/python-termcast-client
Because terminal output is just bytes, we can beam these bytes around!
Further Reading:
Fun Videos
Thanks for listening!
I'll announce on Twitter (@ballingt) when a blog post is up
I'll be at both days of sprints, probably working on bpython and Curtsies


Postcode locationcode is required.
Postcode district is required.
Postcode province is required.
Postcode postcode is required.


|
Panda3D
|
Public Member Functions | |
| def | __init__ (self) |
| def | addAutoInterest (self, parentId, zoneIdList, description) |
| def | addInterest (self, parentId, zoneIdList, description, event=None) |
| def | alterInterest (self, handle, parentId, zoneIdList, description=None, event=None) |
| def | cleanupWaitAllInterestsComplete (self) |
| def | closeAutoInterests (self, obj) |
| def | getAllInterestsCompleteEvent (self) |
| def | handleInterestDoneMessage (self, di) |
| def | isValidInterestHandle (self, handle) |
| def | noNewInterests (self) |
| def | openAutoInterests (self, obj) |
| def | printInterestHistory (self) |
| def | printInterests (self) |
| def | printInterestSets (self) |
| def | printInterestsIfDebug (self) |
| def | queueAllInterestsCompleteEvent (self, frames=5) |
| def | removeAIInterest (self, handle) |
| def | removeAutoInterest (self, handle) |
| def | removeInterest (self, handle, event=None) |
| def | resetInterestStateForConnectionLoss (self) |
| def | setAllInterestsCompleteCallback (self, callback) |
| def | setNoNewInterests (self, flag) |
| def | updateInterestDescription (self, handle, desc) |
Static Public Attributes | |
| dConfigParam | |
| InterestDebug = ConfigVariableBool('interest-debug', False) | |
| notify = directNotify.newCategory("DoInterestManager") | |
| types | |
Top level Interest Manager
| def __init__ | ( | self | ) |
| def addAutoInterest | ( | self, | |
| parentId, | |||
| zoneIdList, | |||
| description | |||
| ) |
Look into a (set of) zone(s).
| def addInterest | ( | self, | |
| parentId, | |||
| zoneIdList, | |||
| description, | |||
event = None |
|||
| ) |
Look into a (set of) zone(s).
| def alterInterest | ( | self, | |
| handle, | |||
| parentId, | |||
| zoneIdList, | |||
description = None , |
|||
event = None |
|||
| ) |
Removes old interests and adds new interests. Note that when an interest is changed, only the most recent change's event will be triggered. Previous events are abandoned. If this is a problem, consider opening multiple interests.
| def cleanupWaitAllInterestsComplete | ( | self | ) |
| def closeAutoInterests | ( | self, | |
| obj | |||
| ) |
| def getAllInterestsCompleteEvent | ( | self | ) |
| def handleInterestDoneMessage | ( | self, | |
| di | |||
| ) |
This handles the interest done messages and may dispatch an event
| def isValidInterestHandle | ( | self, | |
| handle | |||
| ) |
| def noNewInterests | ( | self | ) |
| def openAutoInterests | ( | self, | |
| obj | |||
| ) |
| def printInterestHistory | ( | self | ) |
| def printInterests | ( | self | ) |
| def printInterestSets | ( | self | ) |
| def printInterestsIfDebug | ( | self | ) |
| def queueAllInterestsCompleteEvent | ( | self, | |
frames = 5 |
|||
| ) |
| def removeAIInterest | ( | self, | |
| handle | |||
| ) |
handle is NOT an InterestHandle. It's just a bare integer representing an AI opened interest. We're making the client close down this interest since the AI has trouble removing interests(that its opened) when the avatar goes offline. See GuildManager(UD) for how it's being used.
| def removeAutoInterest | ( | self, | |
| handle | |||
| ) |
Stop looking in a (set of) zone(s)
| def removeInterest | ( | self, | |
| handle, | |||
event = None |
|||
| ) |
Stop looking in a (set of) zone(s)
| def resetInterestStateForConnectionLoss | ( | self | ) |
| def setAllInterestsCompleteCallback | ( | self, | |
| callback | |||
| ) |
| def setNoNewInterests | ( | self, | |
| flag | |||
| ) |
| def updateInterestDescription | ( | self, | |
| handle, | |||
| desc | |||
| ) |
|
static |
|
static |
|
static |
|
static |


Index
A B C D E F G H I K L M N O P R S T U V X Y
All Classes|All Packages
All Classes|All Packages
All Classes|All Packages
P
- pair - Variable in class ship
- place(int, int, int, boolean) - Method in class PlaceShip
-
places ship on board and checks if ship will overlap other ships
- placeAI(Board, BoardPrinterWrapper, PlaceShip) - Static method in class PlaceAIShips
-
places ships on board for AI player
- PlaceAIShips - Class in <Unnamed>
-
PlaceAIShips
- PlaceAIShips() - Constructor for class PlaceAIShips
- placeIt(int, int, int) - Method in class PlaceShip
-
calls placeIterative method to place a ship
- placeIterative(int, int, int) - Method in class PlaceShip
-
actual placing of the ship and making sure the ship in in bounds of the board
- placeShip(Board, int, String) - Method in class Executive
-
This method attempts to place a ship of size shipSize onto the provided Board object.
- PlaceShip - Class in <Unnamed>
-
PlaceShip
- PlaceShip(Board) - Constructor for class PlaceShip
-
PlaceShip Constructor
- placeShipLoop(Board, BoardPrinterWrapper, PlaceShip) - Method in class AIEasy
-
places the ships of the AI player
- placeShipLoop(Board, BoardPrinterWrapper, PlaceShip) - Method in class AIHard
-
places the ships of the AI player
- placeShipLoop(Board, BoardPrinterWrapper, PlaceShip) - Method in class AIMedium
-
places the ships of the AI player
- placeShipLoop(Board, BoardPrinterWrapper, PlaceShip) - Method in interface gameLogicInterface
-
places the ships of the AI player
- placeShipLoop(Board, BoardPrinterWrapper, PlaceShip) - Method in class safelyGetCoordinates
-
Takes the amount and loops that many times to allow the player to place all their ships onto their board
- Play(gameLogicInterface, gameLogicInterface) - Method in class GameLoop
-
Plays the game, calling game running functions in a loop until there is a winner
- player - Variable in class getUserInput
- player1Board - Variable in class GameLoop
- player1place - Variable in class GameLoop
- player1Printer - Variable in class GameLoop
- player1UI - Variable in class GameLoop
- player2Board - Variable in class GameLoop
- player2place - Variable in class GameLoop
- player2Printer - Variable in class GameLoop
- player2UI - Variable in class GameLoop
- playerWon - Variable in class GameLoop
- playerWon(Board) - Method in class Executive
-
This method determines whether a player has won the game.
- print(boolean) - Method in class BoardPrinterWrapper
-
This uses the print function to print a normal board or attack board
- printAndRemove(Board, char) - Static method in class BoardPrinter
-
This static method prints the board without a given character
- printBoard(Board) - Static method in class BoardPrinter
-
This static method prints the board normally
- printList() - Method in class BrokenRadar
-
displays coordinates to player
- printMenu() - Static method in class Utility
-
Displays the menu in the console.
- printStart() - Static method in class Utility
-
Prints out the game welcome messages, starting the game
All Classes|All Packages


Ford Mustang Mach-E Originally Looked Like This
You know a new model is important when Ford makes a mini documentary about it.
by Martin BiggPromotional videos advertising new model launches are nothing new. For the Mustang Mach-E, however, Ford has gone the extra mile and released a mini-documentary video emphasizing the importance of the Blue Oval automaker's first-ever fully electric SUV.
Titled "Making the Mustang Mach-E," the video is narrated by Ford team members and Mustang owners and provides a behind the scenes look at how Ford's first electric SUV was created. If you have 20 minutes to spare, it's well worth a watch, as it covers the design process and the initial controversy surrounding the electric Mustang-inspired SUV.
Originally, Ford's all-electric SUV wasn't going to be called the Mustang and was instead designed to be a technology showcase. "The team was trying to show to the customers that we have this incredible technical capability," Ted Cannis, global director of electrification, explained. "It was going to be a great battery-electric vehicle, but it wasn't going to be an emotional heartbeat. It was just a car."
It was also going to look radically different to the SUV shown at the 2019 LA Auto Show. Early prototype designs are shown in the documentary, which Jim Farley, Ford's president of new business and technology, described as looking like "a science project." The direction changed when Jim Hackett took over as the company's president and CEO in May 2017 and created a dedicated electric vehicle team called Ford Team Edison, which ultimately persuaded Bill Ford to use the Mustang name.
Ford clearly didn't take this decision lightly, as the Mustang name has certain expectations attached to it. Every decision was scrutinized, from the styling and interior to using the iconic pony badge.
The performance also had to live up to the Mustang name. As we discovered, the Mustang Mach-E doesn't disappoint in this area. When it goes on sale next year, the Mustang Mach-E will be available in several flavors, including rear-wheel-drive and all-wheel-drive variants complete with standard- and long-range battery configurations.
With AWD, the standard Mach-E has 332 horsepower and 417 lb-ft of torque, enabling the electric SUV to do 0-62 mph in around 5.5 seconds, while the range-topping GT model increases the output to around 459 hp and reduces the 0-62 mph time to around 3.5 seconds.


vim
|
emacs
sweeten your javascript
/* Welcome to sweet.js! You can play around with macro writing here on the left side and your code will automatically be compiled on the right. This page will also save your code to localStorage on every successful compile so feel free to close the page and come back later! */ // Here is a really simple identity macro to get started. // The `macro` keyword is used to create and name new macros. macro id { rule { // after the macro name, match: // (1) a open paren // (2) a single token and bind it to `$x` // (3) a close paren ($x) } => { // just return the token we bound to `$x` $x } } id ( 42 ); // Note that a single token to sweet.js includes matched // delimiters not just numbers and identifiers. For example, // an array with all of its elements counts as one token: id ([1,2,3]) // One of the really important things sweet.js does is protect // macros from unintentionally binding or capturing variables they // wasn't supposed to. This is called hygiene and to enforce hygiene // sweet.js must carefully rename all variable names. var x; // note the different name on the right ---> // For example, let's look at the swap macro that uses a temporary // variable to swap the contents of two other vars: macro swap { rule { ($x, $y) } => { var tmp = $x; $y = $x; $x = tmp; } } var foo = 100; var bar = 200; var tmp = 'my other temporary variable'; swap (foo, bar) // Notice that even though the `tmp` variable we made outside of // the macro had the same name as the `tmp` variable inside the // macro, sweet.js kept them distinct.


Class ARBFragmentProgram
- java.lang.Object
-
- org.lwjgl.opengl.ARBFragmentProgram
-
public final class ARBFragmentProgram extends java.lang.Object
Native bindings to the ARB_fragment_program extension.Unextended OpenGL mandates a certain set of configurable per- fragment computations defining texture application, texture environment, color sum, and fog operations. Several extensions have added further per-fragment computations to OpenGL. For example, extensions have defined new texture environment capabilities (ARB_texture_env_add,
ARB_texture_env_combine
,ARB_texture_env_dot3
, ARB_texture_env_crossbar), per-fragment depth comparisons (ARB_depth_texture
,ARB_shadow
,ARB_shadow_ambient
, EXT_shadow_funcs), per-fragment lighting (EXT_fragment_lighting, EXT_light_texture), and environment mapped bump mapping (ATI_envmap_bumpmap).Each such extension adds a small set of relatively inflexible per-fragment computations.
This inflexibility is in contrast to the typical flexibility provided by the underlying programmable floating point engines (whether micro-coded fragment engines, DSPs, or CPUs) that are traditionally used to implement OpenGL's texturing computations. The purpose of this extension is to expose to the OpenGL application writer a significant degree of per-fragment programmability for computing fragment parameters.
For the purposes of discussing this extension, a fragment program is a sequence of floating-point 4-component vector operations that determines how a set of program parameters (not specific to an individual fragment) and an input set of per-fragment parameters are transformed to a set of per-fragment result parameters.
The per-fragment computations for standard OpenGL given a particular set of texture and fog application modes (along with any state for extensions defining per-fragment computations) is, in essence, a fragment program. However, the sequence of operations is defined implicitly by the current OpenGL state settings rather than defined explicitly as a sequence of instructions.
This extension provides an explicit mechanism for defining fragment program instruction sequences for application-defined fragment programs. In order to define such fragment programs, this extension defines a fragment programming model including a floating-point 4-component vector instruction set and a relatively large set of floating-point 4-component registers.
The extension's fragment programming model is designed for efficient hardware implementation and to support a wide variety of fragment programs. By design, the entire set of existing fragment programs defined by existing OpenGL per-fragment computation extensions can be implemented using the extension's fragment programming model.
LWJGL: This extension defines many functions and tokens that are also defined in
ARB_vertex_program
. Since these two extensions are often used together, the common functionality has only been exposed byARB_vertex_program
, to avoid static import conflicts.
-
-
Field Summary
Fields Modifier and Type Field and Description static int
GL_FRAGMENT_PROGRAM_ARB
Accepted by thecap
parameter of Disable, Enable, and IsEnabled, by thepname
parameter of GetBooleanv, GetIntegerv, GetFloatv, and GetDoublev, and by thetarget
parameter of ProgramStringARB, BindProgramARB, ProgramEnvParameter4[df][v]ARB, ProgramLocalParameter4[df][v]ARB, GetProgramEnvParameter[df]vARB, GetProgramLocalParameter[df]vARB, GetProgramivARB and GetProgramStringARB.static int
GL_MAX_PROGRAM_ALU_INSTRUCTIONS_ARB
GL_MAX_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB
GL_MAX_PROGRAM_NATIVE_TEX_INDIRECTIONS_ARB
GL_MAX_PROGRAM_NATIVE_TEX_INSTRUCTIONS_ARB
GL_MAX_PROGRAM_TEX_INDIRECTIONS_ARB
GL_MAX_PROGRAM_TEX_INSTRUCTIONS_ARBAccepted by thepname
parameter of GetProgramivARB.static int
GL_MAX_TEXTURE_COORDS_ARB
GL_MAX_TEXTURE_IMAGE_UNITS_ARBAccepted by thepname
parameter of GetBooleanv, GetIntegerv, GetFloatv, and GetDoublev.static int
GL_PROGRAM_ALU_INSTRUCTIONS_ARB
GL_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB
GL_PROGRAM_NATIVE_TEX_INDIRECTIONS_ARB
GL_PROGRAM_NATIVE_TEX_INSTRUCTIONS_ARB
GL_PROGRAM_TEX_INDIRECTIONS_ARB
GL_PROGRAM_TEX_INSTRUCTIONS_ARBAccepted by thepname
parameter of GetProgramivARB.
-
-
-
Field Detail
-
GL_FRAGMENT_PROGRAM_ARB
public static final int GL_FRAGMENT_PROGRAM_ARB
Accepted by thecap
parameter of Disable, Enable, and IsEnabled, by thepname
parameter of GetBooleanv, GetIntegerv, GetFloatv, and GetDoublev, and by thetarget
parameter of ProgramStringARB, BindProgramARB, ProgramEnvParameter4[df][v]ARB, ProgramLocalParameter4[df][v]ARB, GetProgramEnvParameter[df]vARB, GetProgramLocalParameter[df]vARB, GetProgramivARB and GetProgramStringARB.- See Also:
- Constant Field Values
-
GL_PROGRAM_ALU_INSTRUCTIONS_ARB
public static final int GL_PROGRAM_ALU_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_PROGRAM_TEX_INSTRUCTIONS_ARB
public static final int GL_PROGRAM_TEX_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_PROGRAM_TEX_INDIRECTIONS_ARB
public static final int GL_PROGRAM_TEX_INDIRECTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB
public static final int GL_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_PROGRAM_NATIVE_TEX_INSTRUCTIONS_ARB
public static final int GL_PROGRAM_NATIVE_TEX_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_PROGRAM_NATIVE_TEX_INDIRECTIONS_ARB
public static final int GL_PROGRAM_NATIVE_TEX_INDIRECTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_PROGRAM_ALU_INSTRUCTIONS_ARB
public static final int GL_MAX_PROGRAM_ALU_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_PROGRAM_TEX_INSTRUCTIONS_ARB
public static final int GL_MAX_PROGRAM_TEX_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_PROGRAM_TEX_INDIRECTIONS_ARB
public static final int GL_MAX_PROGRAM_TEX_INDIRECTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB
public static final int GL_MAX_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_PROGRAM_NATIVE_TEX_INSTRUCTIONS_ARB
public static final int GL_MAX_PROGRAM_NATIVE_TEX_INSTRUCTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_PROGRAM_NATIVE_TEX_INDIRECTIONS_ARB
public static final int GL_MAX_PROGRAM_NATIVE_TEX_INDIRECTIONS_ARB
Accepted by thepname
parameter of GetProgramivARB.- See Also:
- Constant Field Values
-
GL_MAX_TEXTURE_COORDS_ARB
public static final int GL_MAX_TEXTURE_COORDS_ARB
Accepted by thepname
parameter of GetBooleanv, GetIntegerv, GetFloatv, and GetDoublev.- See Also:
- Constant Field Values
-
GL_MAX_TEXTURE_IMAGE_UNITS_ARB
public static final int GL_MAX_TEXTURE_IMAGE_UNITS_ARB
Accepted by thepname
parameter of GetBooleanv, GetIntegerv, GetFloatv, and GetDoublev.- See Also:
- Constant Field Values
-
-


Home
Start Quiz
Results
Select quiz type
No Of Questions
Category
Any
Genral Knowledge
Books
Films
Music
Difficulty
Any
Easy
Medium
Hard


Personal Projects
GitHub Guides - an indepth tutorial published by GitHub. Perfect for those looking to set up and understand the platform.
HTML Cheat Sheet - A quick refernce guide for HTML.
CSS Cheat Sheet - A quick refercne guide for CSS.
Will soon post my contributions to other projects and projects of my own...


Page maintained by Ke-Sen Huang.
If you have additions or changes, send an e-mail.
Note that when possible I link to the page containing the link to the actual PDF or PS of the preprint.
I prefer this as it gives some context to the paper and avoids possible copyright problems with direct linking.
Thus you may need to search on the page to find the actual document.
the Eurographics Digital Library Link for the paper
Paper Abstract
Author Preprint
Paper Video
Paper Presentation
Paper Images
Paper Data or Auxiliary Material
Demo Program or Source Code
Related Links
Changelog
Optics
- Polynomial Optics: A Toolkit for Efficient Ray-Tracing of Lens Systems
-
Matthias B. Hullin,
Johannes Hanika,
Wolfgang Heidrich
- Per-Vertex Defocus Blur for Stochastic Rasterization
-
Jacob Munkberg,
Robert Toth,
Tomas Akenine-Moller
Global illumination
- Pre-convolved Radiance Caching
-
Daniel Scherzer,
Chuong Nguyen,
Tobias Ritschel,
Hans-Peter Seidel
- Quantized Point-Based Global Illumination
-
Bert Buchholz,
Tamy Boubekeur
- Progressive Virtual Beam Lights
-
Jan Novak,
Derek Nowrouzezahrai,
Carsten Dachsbacher,
Wojciech Jarosz
Image analysis and editing
- Intrinsic Images by Clustering
-
Elena Garces, Adolfo Munoz,
Jorge Lopez-Moreno,
Diego Gutierrez
- Artistic Illumination Transfer for Portraits
-
Xiaowu Chen, Xin Jin, Qinping Zhao, Hongyu Wu
- Matting and Compositing for Fresnel Reflection on Wavy Surfaces
-
Yuki Endo,
Yoshihiro Kanamori,
Yukio Fukui,
Jun Mitani
Geometry
- Geometry Presorting for Implicit Object Space Partitioning
-
Martin Eisemann,
Pablo Bauszat,
Stefan Guthe,
Marcus Magnor
- Parameterization-Aware MIP-Mapping
-
Josiah Manson,
Scott Schaefer
- Tessellation-Independent Smooth Shadow Boundaries
-
Oliver Mattausch,
Daniel Scherzer,
Michael Wimmer,
Takeo Igarashi
Non-Photorealistic Rendering
- A Stylized Approach for Pencil Drawing from Photographs
-
Heekyung Yang,
Yunmi Kwon,
Kyungha Min
- Surface Relief Analysis for Illustrative Shading
-
Lucas Ammann,
Pascal Barla,
Gael Guennebaud,
Xavier Granier,
Patrick Reuter
Material appearance
- A Statistical Method for SVBRDF Approximation from Video Sequence in General Lighting Conditions
-
Gianpaolo Palma,
Marco Callieri,
Matteo Dellepiane,
Roberto Scopigno
- Analytic Tangent Irradiance Environment Maps for Anisotropic Surfaces
-
Soham Mehta,
Ravi Ramamorthi,
Mark Meyer,
Christophe Hery
- Accurate fitting of measured reflectances using a Shifted Gamma micro-facet distribution
-
Mahdi M. Bagher,
Cyril Soler,
,
Nicolas Holzschuch
Sampling
- Importance Sampling Techniques for Path Tracing in Participating Media
-
Christopher Kulla,
Marcos Fajardo
- Fast Generation of Approximate Blue Noise Point Sets
-
Nima Khademi Kalantari,
Pradeep Sen
- ISHair: Importance Sampling for Hair Scattering
()
(
)
-
Jiawei Ou,
Feng Xie,
Parashar Krishnamachari,
Fabio Pellacini
Material synthesis
- Example-Based Fractured Appearance
(
)
()
-
Loeiz Glondu,
Lien Muguercia,
Carles Bosch,
Holly Rushmeier,
Georges Dumont,
George Drettakis
- Gamut Mapping Spatially Varying Reflectance with an Improved BRDF Similarity Metric
-
Thiago Pereira,
Szymon Rusinkiewicz
kesen.huang@gmail.com


WITNESS LIST
Aging
October 2, 2014 11:00 AM
Alzheimer's and related disorders
ON:
Devere, MD, Ronald (Alzheimer's Association), Austin, TX
Ducayet, Patty State Long-Term Care Ombudsman (Ombudsman program and residents
with dementia), Austin, TX
Hanna, Debbie Chair (Texas Council on Alzheimer's and Texas Alzheimer's Research and
Care Consortium), Austin, TX
Weizenbaum, Jon Commissioner (Department of Aging and Disability Services), Austin,
TX
Registering, but not testifying:
ON:
Wells, Christian Executive Director (Alzheimer's Association - Capital of Texas Chapter),
Austin, TX
Area Agencies on Aging
ON:
Cooper, Curtis Manager - Houston/Galveston Area Council (Area Agencies on Aging),
Houston, TX
Lugo, Yvette President (Texas Association of Area Agencies on Aging), El Paso, TX
Moore, Deborah Bureau Chief of Aging (Area Agencies on Aging), Houston, TX
Pitts, Maurice County Commissioner (Texas Association of Regional Councils), Giddings,
TX


themes
At the moment you can find my publications at Hussein Ali Agrama
You can contact me at Hussein Ali Agrama


Changelog (underlined versions were not publicly released)
v<IP_ADDRESS> - 2015-03-29 :
- Massive bugfixes
v<IP_ADDRESS> - 2015-03-27 :
- Added authentication
- Refactored code, optimized a lot
v<IP_ADDRESS> - 2015-03-24 :
- Added client login loop (no authentication)
- Fixed a few bugs and crashes
v<IP_ADDRESS> - 2015-03-22 :
- Updated server commands
- Fixed a few bugs and crashes
v<IP_ADDRESS> - 2015-03-04 :
- Added credentials management, with basic error treatment and update/delete functions [server-side]
v<IP_ADDRESS> - 2015-02-28 :
- Fixed colors
v<IP_ADDRESS> - 2015-02-27 :
- Added dynamic connection
v<IP_ADDRESS> - 2014-12-22 :
- Worked on loggers
- Pre-implemented extensive console control
v0.1.2 - 2014-12-21 :
- Movement is now possible through server communication only
- Added error management
v<IP_ADDRESS> - 2014-11-16 :
- Communication protocol in progress
- Started implementing MongoDB databases
v0.1.1 - 2014-11-09 :
- Worked on server-client communication
v0.1.0 - 2014-10-11 :
- Added nodejs server
- Rewrote most of the code
- Started local development with server features
- Started using mySQL for user data storage instead of raw JSON serialization
v0.0.2 - 2014-10-05 :
- Second-order movement (more or less smooth)
- The main webpage is required to access the game
- Added font support
- Added changelog
- Added multiple keys support
- W3C validated
v0.0.1 - 2014-10-04 :
- Web launch


A
motherboard
, also known as a
logic
board
on Apple Computers, is the primary circuit board making up a modern computer.


Welcome!
Functional Programming
in JavaScript
Wait, what?
- Functions as primary abstraction - building blocks
- Minimize side-effects
- Immutability
- Declarative vs imperative - processing flow of data
Also:
- Lazy/delayed evaluation
- Composition
- Combinators
- Functors
- Applicatives
- Monoids
- Monads
JavaScript: The Best Parts
What'll the meetup cover?
- Functional programming and it's application
- Functional libraries e.g. Underscore/Lo-Dash, Functional.js, Bacon.js
- AltJS languages: LiveScript, Roy, ClojureScript
- ...and more
Submit a talk!
Drop us a line
Tonight:
- JS == Scheme, Tommy Hall
- FP Dojo
Dojo
Site generator for JSGarden
Original code has quite a bit of state - not required for a transform.
Go!
- github.com/timruffles/JavaScript-Garden/tree/fp-dojo
- Node-based site generator, small - ~200 LOC
- Form groups of 3+
- 1 hour, equal turns driving code
- Then we'll talk through code
Ideas for focus
github.com/timruffles/JavaScript-Garden/tree/fp-dojo
- Split IO/pure functions
- Streams
- Lazy enumerators
- Monads
- Function composition, partials
- Clojurescript, LiveScript etc
Enjoy!